{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn import model_selection,preprocessing,ensemble\n",
    "\n",
    "data_df=pd.read_csv(\"train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature1(data, test=False):\n",
    "    if test == 1:\n",
    "        dealed_data = data.drop_duplicates(['file_id'])['file_id']\n",
    "    else:\n",
    "        dealed_data = data.drop_duplicates(['file_id'])[['file_id','label']]\n",
    "    # 提取各类别特征的类别数\n",
    "    dealed_train = data.groupby(\"file_id\")[[\"api\",'tid','return_value']].nunique()\n",
    "    dealed_train.columns = ['api_nunique','tid_nunique','value_nunique']\n",
    "    \n",
    "    temp = data.groupby(['file_id'])['index'].count().rename('id_count')\n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    # 对每个file_id,的每个feat计数，然后求其统计特征\n",
    "    cate_feat = ['api','tid','return_value']\n",
    "    for feat in cate_feat:\n",
    "        temp = data.groupby(['file_id',feat])[feat].count().groupby(['file_id']).agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                            pd.Series.skew,pd.Series.kurt]).add_prefix(feat+'_cnt_')\n",
    "        dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    # 提取交叉特征，每个file_id，每个不同api的tid种类数的统计特征\n",
    "    temp = data.groupby(['file_id','api'])['tid'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('api_tid_')\n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    # 提取交叉特征，每个file_id，每个不同api的return种类数的统计特征\n",
    "    temp = data.groupby(['file_id','api'])['return_value'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('api_value_')                                                               \n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    temp = data.groupby(['file_id','tid'])['api'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('tid_tid_')\n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "\n",
    "    temp = data.groupby(['file_id','tid'])['return_value'].nunique().groupby('file_id').agg(['min','max','mean','median','std',pd.Series.mad,\n",
    "                                                                    pd.Series.skew,pd.Series.kurt]).add_prefix('tid_value_')                                                               \n",
    "    dealed_train = pd.concat([dealed_train, temp],axis=1)\n",
    "    return dealed_train\n",
    "\n",
    "train_data = feature1(data_df, test=False)\n",
    "#test_data = feature1(test, test=True)\n",
    "train_data.to_csv('train_featur_all.csv', index=False)\n",
    "#test_data.to_csv('test_feature1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_text=data_df.groupby(['file_id'])['api'].apply(lambda x:' '.join(list(x))).to_frame().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data=pd.merge(train_data,api_text,how='left', on='file_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X=train_data.iloc[:,:-1]\n",
    "train_Y=data_df.groupby(['file_id','label'])['label'].unique()\n",
    "train_Y=np.array(train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgb_logloss(preds,data):\n",
    "    labels_ = data.get_label()\n",
    "    classes_ = np.unique(labels_) \n",
    "    preds_prob = []\n",
    "    \n",
    "    for i in range(len(classes_)):\n",
    "        preds_prob.append(preds[i*len(labels_):(i+1) * len(labels_)])\n",
    "    #print(preds_prob)\n",
    "    preds_prob_ = np.vstack(preds_prob) \n",
    "    \n",
    "    loss = [] \n",
    "    for i in range(preds_prob_.shape[1]):  # 样本个数\n",
    "        sum_ = 0  \n",
    "        for j in range(preds_prob_.shape[0]): #类别个数\n",
    "            pred = preds_prob_[j,i] # 第i个样本预测为第j类的概率\n",
    "            if  j == labels_[i]:\n",
    "                sum_ += np.log(pred)\n",
    "            else:\n",
    "                sum_ += np.log(1 - pred) \n",
    "             \n",
    "        loss.append(sum_)  \n",
    "         \n",
    "    return 'loss is: ' ,-1 * (np.sum(loss) / preds_prob_.shape[1]),False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_Y, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def runLGB(x_train,y_train,x_valid,y_valid):\n",
    "    dtrain = lgb.Dataset(x_train,y_train) \n",
    "    dval   = lgb.Dataset(x_valid,y_valid, reference = dtrain) \n",
    "\n",
    "    params = {\n",
    "         'task':'train', \n",
    "         'num_leaves': 255,\n",
    "         'objective': 'multiclass',\n",
    "         'num_class':6,\n",
    "        #'min_data_in_leaf': 40,\n",
    "         'min_data_in_leaf': 1,\n",
    "        'learning_rate': 0.05,\n",
    "        'feature_fraction': 0.85,\n",
    "        'bagging_fraction': 0.9,\n",
    "         'bagging_freq': 5, \n",
    "         'max_bin':128,\n",
    "        'num_threads': 10,\n",
    "        'random_state':100\n",
    "     }  \n",
    "    lgb_model_0_order = lgb.train(params, dtrain, num_boost_round=500,valid_sets=[dtrain,dval], early_stopping_rounds=50)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text  import TfidfVectorizer\n",
    "from scipy import sparse\n",
    "import os \n",
    "import pickle\n",
    "n_range=(1,3)  #\n",
    "max_feature=100000 #\n",
    "\n",
    "if os.path.exists(\"api_tfidf.pkl\")==False:\n",
    "    api_tfidf=TfidfVectorizer(ngram_range=n_range,max_features=max_feature)\n",
    "    api_tfidf.fit(train_data['api'])\n",
    "    with open(\"api_tfidf.pkl\",'wb') as f:\n",
    "        pickle.dump(api_tfidf,f)\n",
    "        \n",
    "else:\n",
    "    with open(\"api_tfidf.pkl\",'rb')  as f:\n",
    "        api_tfidf=pickle.load(f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_sparse1=api_tfidf.transform(train_data['api'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 61), (116624, 66540))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.iloc[:,:-1].shape,tr_sparse1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X=tr_sparse1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.50805\ttest-mlogloss:1.50837\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.29799\ttest-mlogloss:1.29853\n",
      "[2]\ttrain-mlogloss:1.13258\ttest-mlogloss:1.13338\n",
      "[3]\ttrain-mlogloss:0.9967\ttest-mlogloss:0.997618\n",
      "[4]\ttrain-mlogloss:0.882631\ttest-mlogloss:0.883652\n",
      "[5]\ttrain-mlogloss:0.785398\ttest-mlogloss:0.78662\n",
      "[6]\ttrain-mlogloss:0.701508\ttest-mlogloss:0.702931\n",
      "[7]\ttrain-mlogloss:0.628052\ttest-mlogloss:0.629634\n",
      "[8]\ttrain-mlogloss:0.56374\ttest-mlogloss:0.565399\n",
      "[9]\ttrain-mlogloss:0.506943\ttest-mlogloss:0.508717\n",
      "[10]\ttrain-mlogloss:0.456703\ttest-mlogloss:0.458619\n",
      "[11]\ttrain-mlogloss:0.412078\ttest-mlogloss:0.414033\n",
      "[12]\ttrain-mlogloss:0.372285\ttest-mlogloss:0.374431\n",
      "[13]\ttrain-mlogloss:0.336921\ttest-mlogloss:0.33919\n",
      "[14]\ttrain-mlogloss:0.305016\ttest-mlogloss:0.307391\n",
      "[15]\ttrain-mlogloss:0.276481\ttest-mlogloss:0.279012\n",
      "[16]\ttrain-mlogloss:0.250891\ttest-mlogloss:0.253565\n",
      "[17]\ttrain-mlogloss:0.227902\ttest-mlogloss:0.230673\n",
      "[18]\ttrain-mlogloss:0.207231\ttest-mlogloss:0.21011\n",
      "[19]\ttrain-mlogloss:0.188583\ttest-mlogloss:0.191655\n",
      "[20]\ttrain-mlogloss:0.171851\ttest-mlogloss:0.175102\n",
      "[21]\ttrain-mlogloss:0.15662\ttest-mlogloss:0.160105\n",
      "[22]\ttrain-mlogloss:0.143092\ttest-mlogloss:0.146722\n",
      "[23]\ttrain-mlogloss:0.130668\ttest-mlogloss:0.134452\n",
      "[24]\ttrain-mlogloss:0.119424\ttest-mlogloss:0.12344\n",
      "[25]\ttrain-mlogloss:0.109344\ttest-mlogloss:0.113494\n",
      "[26]\ttrain-mlogloss:0.100188\ttest-mlogloss:0.104575\n",
      "[27]\ttrain-mlogloss:0.091863\ttest-mlogloss:0.09636\n",
      "[28]\ttrain-mlogloss:0.084355\ttest-mlogloss:0.088956\n",
      "[29]\ttrain-mlogloss:0.07762\ttest-mlogloss:0.082423\n",
      "[30]\ttrain-mlogloss:0.071455\ttest-mlogloss:0.076422\n",
      "[31]\ttrain-mlogloss:0.065893\ttest-mlogloss:0.070994\n",
      "[32]\ttrain-mlogloss:0.060875\ttest-mlogloss:0.066103\n",
      "[33]\ttrain-mlogloss:0.056324\ttest-mlogloss:0.061715\n",
      "[34]\ttrain-mlogloss:0.052182\ttest-mlogloss:0.057731\n",
      "[35]\ttrain-mlogloss:0.048474\ttest-mlogloss:0.054139\n",
      "[36]\ttrain-mlogloss:0.045037\ttest-mlogloss:0.050874\n",
      "[37]\ttrain-mlogloss:0.041892\ttest-mlogloss:0.047887\n",
      "[38]\ttrain-mlogloss:0.039079\ttest-mlogloss:0.045168\n",
      "[39]\ttrain-mlogloss:0.0365\ttest-mlogloss:0.0428\n",
      "[40]\ttrain-mlogloss:0.034105\ttest-mlogloss:0.040528\n",
      "[41]\ttrain-mlogloss:0.031918\ttest-mlogloss:0.038551\n",
      "[42]\ttrain-mlogloss:0.029966\ttest-mlogloss:0.036737\n",
      "[43]\ttrain-mlogloss:0.028127\ttest-mlogloss:0.035072\n",
      "[44]\ttrain-mlogloss:0.026433\ttest-mlogloss:0.033589\n",
      "[45]\ttrain-mlogloss:0.024847\ttest-mlogloss:0.032203\n",
      "[46]\ttrain-mlogloss:0.023427\ttest-mlogloss:0.030948\n",
      "[47]\ttrain-mlogloss:0.02212\ttest-mlogloss:0.029811\n",
      "[48]\ttrain-mlogloss:0.020875\ttest-mlogloss:0.028809\n",
      "[49]\ttrain-mlogloss:0.019782\ttest-mlogloss:0.027891\n",
      "[50]\ttrain-mlogloss:0.01877\ttest-mlogloss:0.027064\n",
      "[51]\ttrain-mlogloss:0.017792\ttest-mlogloss:0.026285\n",
      "[52]\ttrain-mlogloss:0.016889\ttest-mlogloss:0.02555\n",
      "[53]\ttrain-mlogloss:0.016058\ttest-mlogloss:0.024828\n",
      "[54]\ttrain-mlogloss:0.015238\ttest-mlogloss:0.024203\n",
      "[55]\ttrain-mlogloss:0.014522\ttest-mlogloss:0.023684\n",
      "[56]\ttrain-mlogloss:0.01389\ttest-mlogloss:0.023187\n",
      "[57]\ttrain-mlogloss:0.013238\ttest-mlogloss:0.022685\n",
      "[58]\ttrain-mlogloss:0.012623\ttest-mlogloss:0.022243\n",
      "[59]\ttrain-mlogloss:0.012048\ttest-mlogloss:0.021888\n",
      "[60]\ttrain-mlogloss:0.01158\ttest-mlogloss:0.021538\n",
      "[61]\ttrain-mlogloss:0.011126\ttest-mlogloss:0.021243\n",
      "[62]\ttrain-mlogloss:0.010699\ttest-mlogloss:0.020925\n",
      "[63]\ttrain-mlogloss:0.010272\ttest-mlogloss:0.020657\n",
      "[64]\ttrain-mlogloss:0.00988\ttest-mlogloss:0.020433\n",
      "[65]\ttrain-mlogloss:0.009555\ttest-mlogloss:0.020188\n",
      "[66]\ttrain-mlogloss:0.009218\ttest-mlogloss:0.019993\n",
      "[67]\ttrain-mlogloss:0.008894\ttest-mlogloss:0.019833\n",
      "[68]\ttrain-mlogloss:0.008582\ttest-mlogloss:0.019688\n",
      "[69]\ttrain-mlogloss:0.008296\ttest-mlogloss:0.019519\n",
      "[70]\ttrain-mlogloss:0.008044\ttest-mlogloss:0.019396\n",
      "[71]\ttrain-mlogloss:0.007773\ttest-mlogloss:0.019285\n",
      "[72]\ttrain-mlogloss:0.007571\ttest-mlogloss:0.019173\n",
      "[73]\ttrain-mlogloss:0.007337\ttest-mlogloss:0.019005\n",
      "[74]\ttrain-mlogloss:0.007118\ttest-mlogloss:0.018841\n",
      "[75]\ttrain-mlogloss:0.006904\ttest-mlogloss:0.018721\n",
      "[76]\ttrain-mlogloss:0.006725\ttest-mlogloss:0.018662\n",
      "[77]\ttrain-mlogloss:0.006551\ttest-mlogloss:0.018578\n",
      "[78]\ttrain-mlogloss:0.0064\ttest-mlogloss:0.01852\n",
      "[79]\ttrain-mlogloss:0.006252\ttest-mlogloss:0.018444\n",
      "[80]\ttrain-mlogloss:0.006083\ttest-mlogloss:0.018392\n",
      "[81]\ttrain-mlogloss:0.00594\ttest-mlogloss:0.018318\n",
      "[82]\ttrain-mlogloss:0.005811\ttest-mlogloss:0.018287\n",
      "[83]\ttrain-mlogloss:0.005686\ttest-mlogloss:0.01822\n",
      "[84]\ttrain-mlogloss:0.005573\ttest-mlogloss:0.018163\n",
      "[85]\ttrain-mlogloss:0.005438\ttest-mlogloss:0.01812\n",
      "[86]\ttrain-mlogloss:0.005331\ttest-mlogloss:0.018071\n",
      "[87]\ttrain-mlogloss:0.005223\ttest-mlogloss:0.018054\n",
      "[88]\ttrain-mlogloss:0.005135\ttest-mlogloss:0.018018\n",
      "[89]\ttrain-mlogloss:0.005042\ttest-mlogloss:0.018012\n",
      "[90]\ttrain-mlogloss:0.004941\ttest-mlogloss:0.017953\n",
      "[91]\ttrain-mlogloss:0.004854\ttest-mlogloss:0.017909\n",
      "[92]\ttrain-mlogloss:0.004764\ttest-mlogloss:0.017873\n",
      "[93]\ttrain-mlogloss:0.004686\ttest-mlogloss:0.017876\n",
      "[94]\ttrain-mlogloss:0.004604\ttest-mlogloss:0.017874\n",
      "[95]\ttrain-mlogloss:0.004507\ttest-mlogloss:0.017868\n",
      "[96]\ttrain-mlogloss:0.004435\ttest-mlogloss:0.017871\n",
      "[97]\ttrain-mlogloss:0.004363\ttest-mlogloss:0.01787\n",
      "[98]\ttrain-mlogloss:0.004298\ttest-mlogloss:0.017857\n",
      "[99]\ttrain-mlogloss:0.00422\ttest-mlogloss:0.017812\n",
      "[100]\ttrain-mlogloss:0.00415\ttest-mlogloss:0.017795\n",
      "[101]\ttrain-mlogloss:0.00409\ttest-mlogloss:0.017773\n",
      "[102]\ttrain-mlogloss:0.004022\ttest-mlogloss:0.017759\n",
      "[103]\ttrain-mlogloss:0.003963\ttest-mlogloss:0.017754\n",
      "[104]\ttrain-mlogloss:0.003902\ttest-mlogloss:0.017753\n",
      "[105]\ttrain-mlogloss:0.003833\ttest-mlogloss:0.017748\n",
      "[106]\ttrain-mlogloss:0.003768\ttest-mlogloss:0.017735\n",
      "[107]\ttrain-mlogloss:0.003719\ttest-mlogloss:0.017723\n",
      "[108]\ttrain-mlogloss:0.003662\ttest-mlogloss:0.017737\n",
      "[109]\ttrain-mlogloss:0.003621\ttest-mlogloss:0.017748\n",
      "[110]\ttrain-mlogloss:0.003573\ttest-mlogloss:0.017776\n",
      "[111]\ttrain-mlogloss:0.003523\ttest-mlogloss:0.017777\n",
      "[112]\ttrain-mlogloss:0.00349\ttest-mlogloss:0.017768\n",
      "[113]\ttrain-mlogloss:0.003448\ttest-mlogloss:0.017758\n",
      "[114]\ttrain-mlogloss:0.003404\ttest-mlogloss:0.017787\n",
      "[115]\ttrain-mlogloss:0.003363\ttest-mlogloss:0.017778\n",
      "[116]\ttrain-mlogloss:0.003326\ttest-mlogloss:0.017768\n",
      "[117]\ttrain-mlogloss:0.003284\ttest-mlogloss:0.017775\n",
      "[118]\ttrain-mlogloss:0.003244\ttest-mlogloss:0.01778\n",
      "[119]\ttrain-mlogloss:0.003197\ttest-mlogloss:0.017807\n",
      "[120]\ttrain-mlogloss:0.003152\ttest-mlogloss:0.017831\n",
      "[121]\ttrain-mlogloss:0.003119\ttest-mlogloss:0.017844\n",
      "[122]\ttrain-mlogloss:0.003079\ttest-mlogloss:0.017818\n",
      "[123]\ttrain-mlogloss:0.003051\ttest-mlogloss:0.017806\n",
      "[124]\ttrain-mlogloss:0.003018\ttest-mlogloss:0.017802\n",
      "[125]\ttrain-mlogloss:0.002986\ttest-mlogloss:0.017814\n",
      "[126]\ttrain-mlogloss:0.002955\ttest-mlogloss:0.017813\n",
      "[127]\ttrain-mlogloss:0.002918\ttest-mlogloss:0.017826\n",
      "Stopping. Best iteration:\n",
      "[107]\ttrain-mlogloss:0.003719\ttest-mlogloss:0.017723\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text  import TfidfVectorizer\n",
    "from scipy import sparse\n",
    "import os \n",
    "import pickle\n",
    "n_range=(1,2)  #\n",
    "max_feature=100000 #\n",
    "\n",
    "if os.path.exists(\"api_tfidf_bi.pkl\")==False:\n",
    "    api_tfidf=TfidfVectorizer(ngram_range=n_range,max_features=max_feature)\n",
    "    api_tfidf.fit(train_data['api'])\n",
    "    with open(\"api_tfidf_bi.pkl\",'wb') as f:\n",
    "        pickle.dump(api_tfidf,f)\n",
    "        \n",
    "else:\n",
    "    with open(\"api_tfidf_bi.pkl\",'rb')  as f:\n",
    "        api_tfidf=pickle.load(f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_sparse2=api_tfidf.transform(train_data['api'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 61), (116624, 30984))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.iloc[:,:-1].shape,tr_sparse2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X=tr_sparse2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.50851\ttest-mlogloss:1.50905\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.29892\ttest-mlogloss:1.29988\n",
      "[2]\ttrain-mlogloss:1.13528\ttest-mlogloss:1.13657\n",
      "[3]\ttrain-mlogloss:1.00015\ttest-mlogloss:1.00159\n",
      "[4]\ttrain-mlogloss:0.886164\ttest-mlogloss:0.888004\n",
      "[5]\ttrain-mlogloss:0.788786\ttest-mlogloss:0.790848\n",
      "[6]\ttrain-mlogloss:0.704951\ttest-mlogloss:0.707264\n",
      "[7]\ttrain-mlogloss:0.632093\ttest-mlogloss:0.634606\n",
      "[8]\ttrain-mlogloss:0.567653\ttest-mlogloss:0.570403\n",
      "[9]\ttrain-mlogloss:0.510799\ttest-mlogloss:0.51371\n",
      "[10]\ttrain-mlogloss:0.460909\ttest-mlogloss:0.464047\n",
      "[11]\ttrain-mlogloss:0.416631\ttest-mlogloss:0.419938\n",
      "[12]\ttrain-mlogloss:0.37666\ttest-mlogloss:0.38007\n",
      "[13]\ttrain-mlogloss:0.340862\ttest-mlogloss:0.344359\n",
      "[14]\ttrain-mlogloss:0.309351\ttest-mlogloss:0.313028\n",
      "[15]\ttrain-mlogloss:0.280934\ttest-mlogloss:0.284766\n",
      "[16]\ttrain-mlogloss:0.255221\ttest-mlogloss:0.259175\n",
      "[17]\ttrain-mlogloss:0.231903\ttest-mlogloss:0.235953\n",
      "[18]\ttrain-mlogloss:0.211086\ttest-mlogloss:0.215233\n",
      "[19]\ttrain-mlogloss:0.192321\ttest-mlogloss:0.196604\n",
      "[20]\ttrain-mlogloss:0.175343\ttest-mlogloss:0.179843\n",
      "[21]\ttrain-mlogloss:0.160164\ttest-mlogloss:0.164825\n",
      "[22]\ttrain-mlogloss:0.146346\ttest-mlogloss:0.151165\n",
      "[23]\ttrain-mlogloss:0.134208\ttest-mlogloss:0.139201\n",
      "[24]\ttrain-mlogloss:0.122822\ttest-mlogloss:0.127924\n",
      "[25]\ttrain-mlogloss:0.112854\ttest-mlogloss:0.118139\n",
      "[26]\ttrain-mlogloss:0.103717\ttest-mlogloss:0.109121\n",
      "[27]\ttrain-mlogloss:0.09542\ttest-mlogloss:0.1009\n",
      "[28]\ttrain-mlogloss:0.0877\ttest-mlogloss:0.093339\n",
      "[29]\ttrain-mlogloss:0.080881\ttest-mlogloss:0.086593\n",
      "[30]\ttrain-mlogloss:0.074552\ttest-mlogloss:0.080392\n",
      "[31]\ttrain-mlogloss:0.068821\ttest-mlogloss:0.074766\n",
      "[32]\ttrain-mlogloss:0.063757\ttest-mlogloss:0.069855\n",
      "[33]\ttrain-mlogloss:0.059112\ttest-mlogloss:0.065298\n",
      "[34]\ttrain-mlogloss:0.054875\ttest-mlogloss:0.061228\n",
      "[35]\ttrain-mlogloss:0.050967\ttest-mlogloss:0.057447\n",
      "[36]\ttrain-mlogloss:0.047478\ttest-mlogloss:0.054016\n",
      "[37]\ttrain-mlogloss:0.044198\ttest-mlogloss:0.050874\n",
      "[38]\ttrain-mlogloss:0.041323\ttest-mlogloss:0.048116\n",
      "[39]\ttrain-mlogloss:0.038598\ttest-mlogloss:0.045521\n",
      "[40]\ttrain-mlogloss:0.036127\ttest-mlogloss:0.043182\n",
      "[41]\ttrain-mlogloss:0.033801\ttest-mlogloss:0.040978\n",
      "[42]\ttrain-mlogloss:0.031766\ttest-mlogloss:0.039101\n",
      "[43]\ttrain-mlogloss:0.029784\ttest-mlogloss:0.037286\n",
      "[44]\ttrain-mlogloss:0.028078\ttest-mlogloss:0.035756\n",
      "[45]\ttrain-mlogloss:0.026473\ttest-mlogloss:0.034291\n",
      "[46]\ttrain-mlogloss:0.025013\ttest-mlogloss:0.032993\n",
      "[47]\ttrain-mlogloss:0.023622\ttest-mlogloss:0.031891\n",
      "[48]\ttrain-mlogloss:0.022351\ttest-mlogloss:0.030791\n",
      "[49]\ttrain-mlogloss:0.021204\ttest-mlogloss:0.02976\n",
      "[50]\ttrain-mlogloss:0.020128\ttest-mlogloss:0.02881\n",
      "[51]\ttrain-mlogloss:0.019149\ttest-mlogloss:0.027973\n",
      "[52]\ttrain-mlogloss:0.018305\ttest-mlogloss:0.027275\n",
      "[53]\ttrain-mlogloss:0.017448\ttest-mlogloss:0.026606\n",
      "[54]\ttrain-mlogloss:0.016635\ttest-mlogloss:0.025921\n",
      "[55]\ttrain-mlogloss:0.015905\ttest-mlogloss:0.025337\n",
      "[56]\ttrain-mlogloss:0.015166\ttest-mlogloss:0.024755\n",
      "[57]\ttrain-mlogloss:0.01455\ttest-mlogloss:0.024276\n",
      "[58]\ttrain-mlogloss:0.01401\ttest-mlogloss:0.023883\n",
      "[59]\ttrain-mlogloss:0.013423\ttest-mlogloss:0.023403\n",
      "[60]\ttrain-mlogloss:0.012906\ttest-mlogloss:0.023047\n",
      "[61]\ttrain-mlogloss:0.012433\ttest-mlogloss:0.022705\n",
      "[62]\ttrain-mlogloss:0.012005\ttest-mlogloss:0.022401\n",
      "[63]\ttrain-mlogloss:0.0116\ttest-mlogloss:0.02213\n",
      "[64]\ttrain-mlogloss:0.011202\ttest-mlogloss:0.021827\n",
      "[65]\ttrain-mlogloss:0.010817\ttest-mlogloss:0.021591\n",
      "[66]\ttrain-mlogloss:0.010476\ttest-mlogloss:0.021376\n",
      "[67]\ttrain-mlogloss:0.010151\ttest-mlogloss:0.021162\n",
      "[68]\ttrain-mlogloss:0.009839\ttest-mlogloss:0.020926\n",
      "[69]\ttrain-mlogloss:0.009519\ttest-mlogloss:0.020712\n",
      "[70]\ttrain-mlogloss:0.009249\ttest-mlogloss:0.020525\n",
      "[71]\ttrain-mlogloss:0.009001\ttest-mlogloss:0.020422\n",
      "[72]\ttrain-mlogloss:0.008764\ttest-mlogloss:0.020258\n",
      "[73]\ttrain-mlogloss:0.008549\ttest-mlogloss:0.020187\n",
      "[74]\ttrain-mlogloss:0.008345\ttest-mlogloss:0.020099\n",
      "[75]\ttrain-mlogloss:0.008161\ttest-mlogloss:0.01998\n",
      "[76]\ttrain-mlogloss:0.007954\ttest-mlogloss:0.019854\n",
      "[77]\ttrain-mlogloss:0.007792\ttest-mlogloss:0.019762\n",
      "[78]\ttrain-mlogloss:0.00755\ttest-mlogloss:0.019682\n",
      "[79]\ttrain-mlogloss:0.007379\ttest-mlogloss:0.019621\n",
      "[80]\ttrain-mlogloss:0.007219\ttest-mlogloss:0.019529\n",
      "[81]\ttrain-mlogloss:0.007061\ttest-mlogloss:0.019458\n",
      "[82]\ttrain-mlogloss:0.00688\ttest-mlogloss:0.019349\n",
      "[83]\ttrain-mlogloss:0.006732\ttest-mlogloss:0.019253\n",
      "[84]\ttrain-mlogloss:0.006596\ttest-mlogloss:0.019201\n",
      "[85]\ttrain-mlogloss:0.006437\ttest-mlogloss:0.01916\n",
      "[86]\ttrain-mlogloss:0.006294\ttest-mlogloss:0.019127\n",
      "[87]\ttrain-mlogloss:0.006178\ttest-mlogloss:0.019054\n",
      "[88]\ttrain-mlogloss:0.006039\ttest-mlogloss:0.01898\n",
      "[89]\ttrain-mlogloss:0.005932\ttest-mlogloss:0.018919\n",
      "[90]\ttrain-mlogloss:0.005814\ttest-mlogloss:0.018897\n",
      "[91]\ttrain-mlogloss:0.005706\ttest-mlogloss:0.018835\n",
      "[92]\ttrain-mlogloss:0.005611\ttest-mlogloss:0.018803\n",
      "[93]\ttrain-mlogloss:0.005513\ttest-mlogloss:0.018754\n",
      "[94]\ttrain-mlogloss:0.005413\ttest-mlogloss:0.018746\n",
      "[95]\ttrain-mlogloss:0.005324\ttest-mlogloss:0.018715\n",
      "[96]\ttrain-mlogloss:0.005245\ttest-mlogloss:0.018694\n",
      "[97]\ttrain-mlogloss:0.00516\ttest-mlogloss:0.018675\n",
      "[98]\ttrain-mlogloss:0.005063\ttest-mlogloss:0.01865\n",
      "[99]\ttrain-mlogloss:0.004969\ttest-mlogloss:0.018618\n",
      "[100]\ttrain-mlogloss:0.004886\ttest-mlogloss:0.018608\n",
      "[101]\ttrain-mlogloss:0.004781\ttest-mlogloss:0.018575\n",
      "[102]\ttrain-mlogloss:0.004693\ttest-mlogloss:0.018546\n",
      "[103]\ttrain-mlogloss:0.00463\ttest-mlogloss:0.018521\n",
      "[104]\ttrain-mlogloss:0.004555\ttest-mlogloss:0.018501\n",
      "[105]\ttrain-mlogloss:0.004463\ttest-mlogloss:0.01847\n",
      "[106]\ttrain-mlogloss:0.004397\ttest-mlogloss:0.018464\n",
      "[107]\ttrain-mlogloss:0.004336\ttest-mlogloss:0.018454\n",
      "[108]\ttrain-mlogloss:0.00425\ttest-mlogloss:0.018408\n",
      "[109]\ttrain-mlogloss:0.00419\ttest-mlogloss:0.018382\n",
      "[110]\ttrain-mlogloss:0.004136\ttest-mlogloss:0.018363\n",
      "[111]\ttrain-mlogloss:0.004057\ttest-mlogloss:0.018345\n",
      "[112]\ttrain-mlogloss:0.004\ttest-mlogloss:0.018327\n",
      "[113]\ttrain-mlogloss:0.003944\ttest-mlogloss:0.01831\n",
      "[114]\ttrain-mlogloss:0.003882\ttest-mlogloss:0.018284\n",
      "[115]\ttrain-mlogloss:0.003834\ttest-mlogloss:0.018281\n",
      "[116]\ttrain-mlogloss:0.003787\ttest-mlogloss:0.018289\n",
      "[117]\ttrain-mlogloss:0.003741\ttest-mlogloss:0.018288\n",
      "[118]\ttrain-mlogloss:0.003695\ttest-mlogloss:0.018317\n",
      "[119]\ttrain-mlogloss:0.003651\ttest-mlogloss:0.018293\n",
      "[120]\ttrain-mlogloss:0.003607\ttest-mlogloss:0.018267\n",
      "[121]\ttrain-mlogloss:0.003555\ttest-mlogloss:0.018279\n",
      "[122]\ttrain-mlogloss:0.003519\ttest-mlogloss:0.018282\n",
      "[123]\ttrain-mlogloss:0.003483\ttest-mlogloss:0.018281\n",
      "[124]\ttrain-mlogloss:0.003439\ttest-mlogloss:0.018288\n",
      "[125]\ttrain-mlogloss:0.003408\ttest-mlogloss:0.018285\n",
      "[126]\ttrain-mlogloss:0.003376\ttest-mlogloss:0.018307\n",
      "[127]\ttrain-mlogloss:0.00334\ttest-mlogloss:0.01831\n",
      "[128]\ttrain-mlogloss:0.003304\ttest-mlogloss:0.018329\n",
      "[129]\ttrain-mlogloss:0.003271\ttest-mlogloss:0.018329\n",
      "[130]\ttrain-mlogloss:0.003234\ttest-mlogloss:0.018313\n",
      "[131]\ttrain-mlogloss:0.0032\ttest-mlogloss:0.018307\n",
      "[132]\ttrain-mlogloss:0.003171\ttest-mlogloss:0.01833\n",
      "[133]\ttrain-mlogloss:0.003133\ttest-mlogloss:0.018345\n",
      "[134]\ttrain-mlogloss:0.003096\ttest-mlogloss:0.018356\n",
      "[135]\ttrain-mlogloss:0.003068\ttest-mlogloss:0.018363\n",
      "[136]\ttrain-mlogloss:0.003042\ttest-mlogloss:0.018365\n",
      "[137]\ttrain-mlogloss:0.003011\ttest-mlogloss:0.018372\n",
      "[138]\ttrain-mlogloss:0.002981\ttest-mlogloss:0.018385\n",
      "[139]\ttrain-mlogloss:0.002954\ttest-mlogloss:0.018384\n",
      "[140]\ttrain-mlogloss:0.002921\ttest-mlogloss:0.018395\n",
      "Stopping. Best iteration:\n",
      "[120]\ttrain-mlogloss:0.003607\ttest-mlogloss:0.018267\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text  import TfidfVectorizer\n",
    "from scipy import sparse\n",
    "import os \n",
    "import pickle\n",
    "n_range=(1,1)  #\n",
    "max_feature=100000 #\n",
    "\n",
    "if os.path.exists(\"api_tfidf_uni.pkl\")==False:\n",
    "    api_tfidf=TfidfVectorizer(ngram_range=n_range,max_features=max_feature)\n",
    "    api_tfidf.fit(train_data['api'])\n",
    "    with open(\"api_tfidf_uni.pkl\",'wb') as f:\n",
    "        pickle.dump(api_tfidf,f)\n",
    "        \n",
    "else:\n",
    "    with open(\"api_tfidf_uni.pkl\",'rb')  as f:\n",
    "        api_tfidf=pickle.load(f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-97d79bd0e1df>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtr_sparse3\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mapi_tfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'api'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'train_data' is not defined"
     ]
    }
   ],
   "source": [
    "tr_sparse3=api_tfidf.transform(train_data['api'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-276d3fe7a9f8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtr_sparse3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'train_data' is not defined"
     ]
    }
   ],
   "source": [
    "train_data.iloc[:,:-1].shape,tr_sparse3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X=tr_sparse3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.51289\ttest-mlogloss:1.51329\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.30644\ttest-mlogloss:1.30732\n",
      "[2]\ttrain-mlogloss:1.14433\ttest-mlogloss:1.14557\n",
      "[3]\ttrain-mlogloss:1.01166\ttest-mlogloss:1.01318\n",
      "[4]\ttrain-mlogloss:0.899392\ttest-mlogloss:0.901356\n",
      "[5]\ttrain-mlogloss:0.804084\ttest-mlogloss:0.80635\n",
      "[6]\ttrain-mlogloss:0.720769\ttest-mlogloss:0.723242\n",
      "[7]\ttrain-mlogloss:0.648298\ttest-mlogloss:0.651071\n",
      "[8]\ttrain-mlogloss:0.585099\ttest-mlogloss:0.588312\n",
      "[9]\ttrain-mlogloss:0.529488\ttest-mlogloss:0.533075\n",
      "[10]\ttrain-mlogloss:0.479746\ttest-mlogloss:0.483658\n",
      "[11]\ttrain-mlogloss:0.435416\ttest-mlogloss:0.439556\n",
      "[12]\ttrain-mlogloss:0.39594\ttest-mlogloss:0.400302\n",
      "[13]\ttrain-mlogloss:0.360626\ttest-mlogloss:0.36525\n",
      "[14]\ttrain-mlogloss:0.328954\ttest-mlogloss:0.333861\n",
      "[15]\ttrain-mlogloss:0.300908\ttest-mlogloss:0.306052\n",
      "[16]\ttrain-mlogloss:0.275255\ttest-mlogloss:0.280547\n",
      "[17]\ttrain-mlogloss:0.252581\ttest-mlogloss:0.258231\n",
      "[18]\ttrain-mlogloss:0.232037\ttest-mlogloss:0.237951\n",
      "[19]\ttrain-mlogloss:0.213145\ttest-mlogloss:0.219328\n",
      "[20]\ttrain-mlogloss:0.196136\ttest-mlogloss:0.202527\n",
      "[21]\ttrain-mlogloss:0.180692\ttest-mlogloss:0.187316\n",
      "[22]\ttrain-mlogloss:0.16702\ttest-mlogloss:0.173827\n",
      "[23]\ttrain-mlogloss:0.154076\ttest-mlogloss:0.161085\n",
      "[24]\ttrain-mlogloss:0.142443\ttest-mlogloss:0.149796\n",
      "[25]\ttrain-mlogloss:0.132198\ttest-mlogloss:0.139795\n",
      "[26]\ttrain-mlogloss:0.122861\ttest-mlogloss:0.130651\n",
      "[27]\ttrain-mlogloss:0.114368\ttest-mlogloss:0.122346\n",
      "[28]\ttrain-mlogloss:0.106531\ttest-mlogloss:0.1148\n",
      "[29]\ttrain-mlogloss:0.099442\ttest-mlogloss:0.107988\n",
      "[30]\ttrain-mlogloss:0.093057\ttest-mlogloss:0.101756\n",
      "[31]\ttrain-mlogloss:0.087106\ttest-mlogloss:0.096042\n",
      "[32]\ttrain-mlogloss:0.081632\ttest-mlogloss:0.090704\n",
      "[33]\ttrain-mlogloss:0.076787\ttest-mlogloss:0.085995\n",
      "[34]\ttrain-mlogloss:0.072189\ttest-mlogloss:0.081626\n",
      "[35]\ttrain-mlogloss:0.068024\ttest-mlogloss:0.077665\n",
      "[36]\ttrain-mlogloss:0.064373\ttest-mlogloss:0.074207\n",
      "[37]\ttrain-mlogloss:0.060977\ttest-mlogloss:0.070954\n",
      "[38]\ttrain-mlogloss:0.057662\ttest-mlogloss:0.067947\n",
      "[39]\ttrain-mlogloss:0.054749\ttest-mlogloss:0.065145\n",
      "[40]\ttrain-mlogloss:0.052029\ttest-mlogloss:0.062632\n",
      "[41]\ttrain-mlogloss:0.049606\ttest-mlogloss:0.060329\n",
      "[42]\ttrain-mlogloss:0.047408\ttest-mlogloss:0.058264\n",
      "[43]\ttrain-mlogloss:0.045375\ttest-mlogloss:0.056428\n",
      "[44]\ttrain-mlogloss:0.043507\ttest-mlogloss:0.054806\n",
      "[45]\ttrain-mlogloss:0.041785\ttest-mlogloss:0.053393\n",
      "[46]\ttrain-mlogloss:0.040108\ttest-mlogloss:0.05193\n",
      "[47]\ttrain-mlogloss:0.038602\ttest-mlogloss:0.050541\n",
      "[48]\ttrain-mlogloss:0.037154\ttest-mlogloss:0.049245\n",
      "[49]\ttrain-mlogloss:0.035825\ttest-mlogloss:0.048069\n",
      "[50]\ttrain-mlogloss:0.034534\ttest-mlogloss:0.046883\n",
      "[51]\ttrain-mlogloss:0.033386\ttest-mlogloss:0.045921\n",
      "[52]\ttrain-mlogloss:0.032333\ttest-mlogloss:0.045013\n",
      "[53]\ttrain-mlogloss:0.031385\ttest-mlogloss:0.044193\n",
      "[54]\ttrain-mlogloss:0.030478\ttest-mlogloss:0.043381\n",
      "[55]\ttrain-mlogloss:0.029673\ttest-mlogloss:0.04271\n",
      "[56]\ttrain-mlogloss:0.028928\ttest-mlogloss:0.042165\n",
      "[57]\ttrain-mlogloss:0.028068\ttest-mlogloss:0.041529\n",
      "[58]\ttrain-mlogloss:0.02732\ttest-mlogloss:0.04089\n",
      "[59]\ttrain-mlogloss:0.026602\ttest-mlogloss:0.040348\n",
      "[60]\ttrain-mlogloss:0.026036\ttest-mlogloss:0.039908\n",
      "[61]\ttrain-mlogloss:0.025436\ttest-mlogloss:0.039438\n",
      "[62]\ttrain-mlogloss:0.024951\ttest-mlogloss:0.039035\n",
      "[63]\ttrain-mlogloss:0.024404\ttest-mlogloss:0.03865\n",
      "[64]\ttrain-mlogloss:0.023874\ttest-mlogloss:0.038283\n",
      "[65]\ttrain-mlogloss:0.023482\ttest-mlogloss:0.037997\n",
      "[66]\ttrain-mlogloss:0.023016\ttest-mlogloss:0.037688\n",
      "[67]\ttrain-mlogloss:0.022642\ttest-mlogloss:0.037416\n",
      "[68]\ttrain-mlogloss:0.022136\ttest-mlogloss:0.037067\n",
      "[69]\ttrain-mlogloss:0.021744\ttest-mlogloss:0.036787\n",
      "[70]\ttrain-mlogloss:0.021381\ttest-mlogloss:0.036535\n",
      "[71]\ttrain-mlogloss:0.021035\ttest-mlogloss:0.03629\n",
      "[72]\ttrain-mlogloss:0.020728\ttest-mlogloss:0.036112\n",
      "[73]\ttrain-mlogloss:0.020383\ttest-mlogloss:0.035906\n",
      "[74]\ttrain-mlogloss:0.020025\ttest-mlogloss:0.03567\n",
      "[75]\ttrain-mlogloss:0.019708\ttest-mlogloss:0.035472\n",
      "[76]\ttrain-mlogloss:0.019448\ttest-mlogloss:0.035276\n",
      "[77]\ttrain-mlogloss:0.019199\ttest-mlogloss:0.035106\n",
      "[78]\ttrain-mlogloss:0.018864\ttest-mlogloss:0.034906\n",
      "[79]\ttrain-mlogloss:0.018644\ttest-mlogloss:0.034825\n",
      "[80]\ttrain-mlogloss:0.018419\ttest-mlogloss:0.034655\n",
      "[81]\ttrain-mlogloss:0.018136\ttest-mlogloss:0.034521\n",
      "[82]\ttrain-mlogloss:0.017871\ttest-mlogloss:0.034405\n",
      "[83]\ttrain-mlogloss:0.017566\ttest-mlogloss:0.034211\n",
      "[84]\ttrain-mlogloss:0.017338\ttest-mlogloss:0.034092\n",
      "[85]\ttrain-mlogloss:0.017075\ttest-mlogloss:0.034001\n",
      "[86]\ttrain-mlogloss:0.016851\ttest-mlogloss:0.033851\n",
      "[87]\ttrain-mlogloss:0.016602\ttest-mlogloss:0.033736\n",
      "[88]\ttrain-mlogloss:0.016419\ttest-mlogloss:0.033602\n",
      "[89]\ttrain-mlogloss:0.016228\ttest-mlogloss:0.033538\n",
      "[90]\ttrain-mlogloss:0.01609\ttest-mlogloss:0.033502\n",
      "[91]\ttrain-mlogloss:0.015932\ttest-mlogloss:0.033408\n",
      "[92]\ttrain-mlogloss:0.015714\ttest-mlogloss:0.033296\n",
      "[93]\ttrain-mlogloss:0.015523\ttest-mlogloss:0.033182\n",
      "[94]\ttrain-mlogloss:0.015367\ttest-mlogloss:0.033021\n",
      "[95]\ttrain-mlogloss:0.015226\ttest-mlogloss:0.032955\n",
      "[96]\ttrain-mlogloss:0.015055\ttest-mlogloss:0.032881\n",
      "[97]\ttrain-mlogloss:0.014913\ttest-mlogloss:0.03283\n",
      "[98]\ttrain-mlogloss:0.014768\ttest-mlogloss:0.03271\n",
      "[99]\ttrain-mlogloss:0.01457\ttest-mlogloss:0.032646\n",
      "[100]\ttrain-mlogloss:0.014443\ttest-mlogloss:0.032619\n",
      "[101]\ttrain-mlogloss:0.014312\ttest-mlogloss:0.032553\n",
      "[102]\ttrain-mlogloss:0.014165\ttest-mlogloss:0.032467\n",
      "[103]\ttrain-mlogloss:0.014029\ttest-mlogloss:0.032399\n",
      "[104]\ttrain-mlogloss:0.013893\ttest-mlogloss:0.032334\n",
      "[105]\ttrain-mlogloss:0.013718\ttest-mlogloss:0.032303\n",
      "[106]\ttrain-mlogloss:0.013614\ttest-mlogloss:0.032261\n",
      "[107]\ttrain-mlogloss:0.013418\ttest-mlogloss:0.032181\n",
      "[108]\ttrain-mlogloss:0.01332\ttest-mlogloss:0.032092\n",
      "[109]\ttrain-mlogloss:0.013159\ttest-mlogloss:0.031962\n",
      "[110]\ttrain-mlogloss:0.013046\ttest-mlogloss:0.031933\n",
      "[111]\ttrain-mlogloss:0.012945\ttest-mlogloss:0.031882\n",
      "[112]\ttrain-mlogloss:0.012779\ttest-mlogloss:0.031785\n",
      "[113]\ttrain-mlogloss:0.012675\ttest-mlogloss:0.031751\n",
      "[114]\ttrain-mlogloss:0.012511\ttest-mlogloss:0.031647\n",
      "[115]\ttrain-mlogloss:0.012394\ttest-mlogloss:0.031624\n",
      "[116]\ttrain-mlogloss:0.012281\ttest-mlogloss:0.031576\n",
      "[117]\ttrain-mlogloss:0.01216\ttest-mlogloss:0.031535\n",
      "[118]\ttrain-mlogloss:0.012071\ttest-mlogloss:0.031508\n",
      "[119]\ttrain-mlogloss:0.011988\ttest-mlogloss:0.031507\n",
      "[120]\ttrain-mlogloss:0.011893\ttest-mlogloss:0.031505\n",
      "[121]\ttrain-mlogloss:0.011807\ttest-mlogloss:0.031497\n",
      "[122]\ttrain-mlogloss:0.011704\ttest-mlogloss:0.031435\n",
      "[123]\ttrain-mlogloss:0.011599\ttest-mlogloss:0.031399\n",
      "[124]\ttrain-mlogloss:0.011502\ttest-mlogloss:0.031413\n",
      "[125]\ttrain-mlogloss:0.011433\ttest-mlogloss:0.031437\n",
      "[126]\ttrain-mlogloss:0.011331\ttest-mlogloss:0.031384\n",
      "[127]\ttrain-mlogloss:0.011255\ttest-mlogloss:0.031347\n",
      "[128]\ttrain-mlogloss:0.011155\ttest-mlogloss:0.031321\n",
      "[129]\ttrain-mlogloss:0.011062\ttest-mlogloss:0.031288\n",
      "[130]\ttrain-mlogloss:0.010942\ttest-mlogloss:0.031207\n",
      "[131]\ttrain-mlogloss:0.010865\ttest-mlogloss:0.031202\n",
      "[132]\ttrain-mlogloss:0.010765\ttest-mlogloss:0.031193\n",
      "[133]\ttrain-mlogloss:0.010693\ttest-mlogloss:0.031175\n",
      "[134]\ttrain-mlogloss:0.010608\ttest-mlogloss:0.031144\n",
      "[135]\ttrain-mlogloss:0.01053\ttest-mlogloss:0.031157\n",
      "[136]\ttrain-mlogloss:0.010439\ttest-mlogloss:0.031108\n",
      "[137]\ttrain-mlogloss:0.010318\ttest-mlogloss:0.031079\n",
      "[138]\ttrain-mlogloss:0.01023\ttest-mlogloss:0.031007\n",
      "[139]\ttrain-mlogloss:0.010152\ttest-mlogloss:0.031006\n",
      "[140]\ttrain-mlogloss:0.010075\ttest-mlogloss:0.030981\n",
      "[141]\ttrain-mlogloss:0.010003\ttest-mlogloss:0.030971\n",
      "[142]\ttrain-mlogloss:0.009937\ttest-mlogloss:0.030959\n",
      "[143]\ttrain-mlogloss:0.009874\ttest-mlogloss:0.030957\n",
      "[144]\ttrain-mlogloss:0.009789\ttest-mlogloss:0.030991\n",
      "[145]\ttrain-mlogloss:0.009718\ttest-mlogloss:0.030975\n",
      "[146]\ttrain-mlogloss:0.009634\ttest-mlogloss:0.030977\n",
      "[147]\ttrain-mlogloss:0.009569\ttest-mlogloss:0.030963\n",
      "[148]\ttrain-mlogloss:0.009491\ttest-mlogloss:0.030927\n",
      "[149]\ttrain-mlogloss:0.009406\ttest-mlogloss:0.030911\n",
      "[150]\ttrain-mlogloss:0.009348\ttest-mlogloss:0.030926\n",
      "[151]\ttrain-mlogloss:0.009302\ttest-mlogloss:0.030918\n",
      "[152]\ttrain-mlogloss:0.009237\ttest-mlogloss:0.030892\n",
      "[153]\ttrain-mlogloss:0.009172\ttest-mlogloss:0.030881\n",
      "[154]\ttrain-mlogloss:0.009116\ttest-mlogloss:0.030856\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[155]\ttrain-mlogloss:0.009039\ttest-mlogloss:0.030893\n",
      "[156]\ttrain-mlogloss:0.008966\ttest-mlogloss:0.03092\n",
      "[157]\ttrain-mlogloss:0.008897\ttest-mlogloss:0.030909\n",
      "[158]\ttrain-mlogloss:0.008843\ttest-mlogloss:0.030908\n",
      "[159]\ttrain-mlogloss:0.008792\ttest-mlogloss:0.030883\n",
      "[160]\ttrain-mlogloss:0.008729\ttest-mlogloss:0.03088\n",
      "[161]\ttrain-mlogloss:0.008678\ttest-mlogloss:0.03089\n",
      "[162]\ttrain-mlogloss:0.008643\ttest-mlogloss:0.030873\n",
      "[163]\ttrain-mlogloss:0.008595\ttest-mlogloss:0.030889\n",
      "[164]\ttrain-mlogloss:0.008542\ttest-mlogloss:0.030874\n",
      "[165]\ttrain-mlogloss:0.00848\ttest-mlogloss:0.030846\n",
      "[166]\ttrain-mlogloss:0.008441\ttest-mlogloss:0.030844\n",
      "[167]\ttrain-mlogloss:0.008389\ttest-mlogloss:0.030838\n",
      "[168]\ttrain-mlogloss:0.008349\ttest-mlogloss:0.030858\n",
      "[169]\ttrain-mlogloss:0.008308\ttest-mlogloss:0.030882\n",
      "[170]\ttrain-mlogloss:0.008251\ttest-mlogloss:0.030867\n",
      "[171]\ttrain-mlogloss:0.008194\ttest-mlogloss:0.030861\n",
      "[172]\ttrain-mlogloss:0.008125\ttest-mlogloss:0.030842\n",
      "[173]\ttrain-mlogloss:0.00808\ttest-mlogloss:0.030829\n",
      "[174]\ttrain-mlogloss:0.008032\ttest-mlogloss:0.030826\n",
      "[175]\ttrain-mlogloss:0.00798\ttest-mlogloss:0.03079\n",
      "[176]\ttrain-mlogloss:0.007938\ttest-mlogloss:0.030815\n",
      "[177]\ttrain-mlogloss:0.007883\ttest-mlogloss:0.030789\n",
      "[178]\ttrain-mlogloss:0.007841\ttest-mlogloss:0.030787\n",
      "[179]\ttrain-mlogloss:0.007798\ttest-mlogloss:0.030788\n",
      "[180]\ttrain-mlogloss:0.007752\ttest-mlogloss:0.030784\n",
      "[181]\ttrain-mlogloss:0.007722\ttest-mlogloss:0.030781\n",
      "[182]\ttrain-mlogloss:0.007663\ttest-mlogloss:0.030777\n",
      "[183]\ttrain-mlogloss:0.007617\ttest-mlogloss:0.030769\n",
      "[184]\ttrain-mlogloss:0.007579\ttest-mlogloss:0.030782\n",
      "[185]\ttrain-mlogloss:0.007539\ttest-mlogloss:0.030768\n",
      "[186]\ttrain-mlogloss:0.007494\ttest-mlogloss:0.030767\n",
      "[187]\ttrain-mlogloss:0.007469\ttest-mlogloss:0.030794\n",
      "[188]\ttrain-mlogloss:0.007437\ttest-mlogloss:0.030804\n",
      "[189]\ttrain-mlogloss:0.007387\ttest-mlogloss:0.030831\n",
      "[190]\ttrain-mlogloss:0.007344\ttest-mlogloss:0.030802\n",
      "[191]\ttrain-mlogloss:0.007322\ttest-mlogloss:0.03082\n",
      "[192]\ttrain-mlogloss:0.007285\ttest-mlogloss:0.030849\n",
      "[193]\ttrain-mlogloss:0.007245\ttest-mlogloss:0.030834\n",
      "[194]\ttrain-mlogloss:0.00719\ttest-mlogloss:0.03081\n",
      "[195]\ttrain-mlogloss:0.007146\ttest-mlogloss:0.030768\n",
      "[196]\ttrain-mlogloss:0.007122\ttest-mlogloss:0.030778\n",
      "[197]\ttrain-mlogloss:0.007086\ttest-mlogloss:0.030763\n",
      "[198]\ttrain-mlogloss:0.007048\ttest-mlogloss:0.030761\n",
      "[199]\ttrain-mlogloss:0.00701\ttest-mlogloss:0.03077\n",
      "[200]\ttrain-mlogloss:0.006985\ttest-mlogloss:0.030783\n",
      "[201]\ttrain-mlogloss:0.006934\ttest-mlogloss:0.030795\n",
      "[202]\ttrain-mlogloss:0.006892\ttest-mlogloss:0.030776\n",
      "[203]\ttrain-mlogloss:0.006858\ttest-mlogloss:0.030787\n",
      "[204]\ttrain-mlogloss:0.006818\ttest-mlogloss:0.030761\n",
      "[205]\ttrain-mlogloss:0.006779\ttest-mlogloss:0.030782\n",
      "[206]\ttrain-mlogloss:0.006749\ttest-mlogloss:0.030797\n",
      "[207]\ttrain-mlogloss:0.006716\ttest-mlogloss:0.030815\n",
      "[208]\ttrain-mlogloss:0.006693\ttest-mlogloss:0.03082\n",
      "[209]\ttrain-mlogloss:0.006648\ttest-mlogloss:0.030811\n",
      "[210]\ttrain-mlogloss:0.006621\ttest-mlogloss:0.030805\n",
      "[211]\ttrain-mlogloss:0.006593\ttest-mlogloss:0.030813\n",
      "[212]\ttrain-mlogloss:0.006562\ttest-mlogloss:0.030769\n",
      "[213]\ttrain-mlogloss:0.006525\ttest-mlogloss:0.030777\n",
      "[214]\ttrain-mlogloss:0.006486\ttest-mlogloss:0.030775\n",
      "[215]\ttrain-mlogloss:0.006462\ttest-mlogloss:0.030787\n",
      "[216]\ttrain-mlogloss:0.006422\ttest-mlogloss:0.030796\n",
      "[217]\ttrain-mlogloss:0.006391\ttest-mlogloss:0.030773\n",
      "[218]\ttrain-mlogloss:0.006356\ttest-mlogloss:0.030757\n",
      "[219]\ttrain-mlogloss:0.006313\ttest-mlogloss:0.030753\n",
      "[220]\ttrain-mlogloss:0.00628\ttest-mlogloss:0.030794\n",
      "[221]\ttrain-mlogloss:0.006243\ttest-mlogloss:0.030793\n",
      "[222]\ttrain-mlogloss:0.006216\ttest-mlogloss:0.03079\n",
      "[223]\ttrain-mlogloss:0.006197\ttest-mlogloss:0.030799\n",
      "[224]\ttrain-mlogloss:0.006173\ttest-mlogloss:0.030813\n",
      "[225]\ttrain-mlogloss:0.006158\ttest-mlogloss:0.030849\n",
      "[226]\ttrain-mlogloss:0.006136\ttest-mlogloss:0.030835\n",
      "[227]\ttrain-mlogloss:0.006115\ttest-mlogloss:0.03084\n",
      "[228]\ttrain-mlogloss:0.006086\ttest-mlogloss:0.030863\n",
      "[229]\ttrain-mlogloss:0.006055\ttest-mlogloss:0.030853\n",
      "[230]\ttrain-mlogloss:0.00603\ttest-mlogloss:0.030854\n",
      "[231]\ttrain-mlogloss:0.006012\ttest-mlogloss:0.030868\n",
      "[232]\ttrain-mlogloss:0.005991\ttest-mlogloss:0.030875\n",
      "[233]\ttrain-mlogloss:0.005962\ttest-mlogloss:0.030874\n",
      "[234]\ttrain-mlogloss:0.005931\ttest-mlogloss:0.030882\n",
      "[235]\ttrain-mlogloss:0.005915\ttest-mlogloss:0.030883\n",
      "[236]\ttrain-mlogloss:0.005895\ttest-mlogloss:0.030898\n",
      "[237]\ttrain-mlogloss:0.00587\ttest-mlogloss:0.030933\n",
      "[238]\ttrain-mlogloss:0.005841\ttest-mlogloss:0.030946\n",
      "[239]\ttrain-mlogloss:0.005821\ttest-mlogloss:0.030977\n",
      "Stopping. Best iteration:\n",
      "[219]\ttrain-mlogloss:0.006313\ttest-mlogloss:0.030753\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = sparse.hstack([train_data.iloc[:,:-1],tr_sparse1]).tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 66601), (116624,))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.50806\ttest-mlogloss:1.50849\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.29816\ttest-mlogloss:1.29873\n",
      "[2]\ttrain-mlogloss:1.13231\ttest-mlogloss:1.13304\n",
      "[3]\ttrain-mlogloss:0.99616\ttest-mlogloss:0.997115\n",
      "[4]\ttrain-mlogloss:0.882032\ttest-mlogloss:0.883126\n",
      "[5]\ttrain-mlogloss:0.785154\ttest-mlogloss:0.786455\n",
      "[6]\ttrain-mlogloss:0.701466\ttest-mlogloss:0.702882\n",
      "[7]\ttrain-mlogloss:0.628036\ttest-mlogloss:0.629566\n",
      "[8]\ttrain-mlogloss:0.563555\ttest-mlogloss:0.565206\n",
      "[9]\ttrain-mlogloss:0.506791\ttest-mlogloss:0.508607\n",
      "[10]\ttrain-mlogloss:0.45644\ttest-mlogloss:0.458398\n",
      "[11]\ttrain-mlogloss:0.411809\ttest-mlogloss:0.413924\n",
      "[12]\ttrain-mlogloss:0.371985\ttest-mlogloss:0.374213\n",
      "[13]\ttrain-mlogloss:0.336451\ttest-mlogloss:0.338828\n",
      "[14]\ttrain-mlogloss:0.304668\ttest-mlogloss:0.307161\n",
      "[15]\ttrain-mlogloss:0.276186\ttest-mlogloss:0.27887\n",
      "[16]\ttrain-mlogloss:0.250466\ttest-mlogloss:0.253373\n",
      "[17]\ttrain-mlogloss:0.22731\ttest-mlogloss:0.230391\n",
      "[18]\ttrain-mlogloss:0.206664\ttest-mlogloss:0.20996\n",
      "[19]\ttrain-mlogloss:0.187883\ttest-mlogloss:0.191332\n",
      "[20]\ttrain-mlogloss:0.170962\ttest-mlogloss:0.174503\n",
      "[21]\ttrain-mlogloss:0.155664\ttest-mlogloss:0.159378\n",
      "[22]\ttrain-mlogloss:0.141901\ttest-mlogloss:0.145803\n",
      "[23]\ttrain-mlogloss:0.12939\ttest-mlogloss:0.133531\n",
      "[24]\ttrain-mlogloss:0.118089\ttest-mlogloss:0.122434\n",
      "[25]\ttrain-mlogloss:0.107917\ttest-mlogloss:0.112454\n",
      "[26]\ttrain-mlogloss:0.09868\ttest-mlogloss:0.103411\n",
      "[27]\ttrain-mlogloss:0.090297\ttest-mlogloss:0.095259\n",
      "[28]\ttrain-mlogloss:0.082772\ttest-mlogloss:0.087816\n",
      "[29]\ttrain-mlogloss:0.075828\ttest-mlogloss:0.08112\n",
      "[30]\ttrain-mlogloss:0.069609\ttest-mlogloss:0.075068\n",
      "[31]\ttrain-mlogloss:0.063989\ttest-mlogloss:0.069641\n",
      "[32]\ttrain-mlogloss:0.058809\ttest-mlogloss:0.064663\n",
      "[33]\ttrain-mlogloss:0.054148\ttest-mlogloss:0.060243\n",
      "[34]\ttrain-mlogloss:0.0499\ttest-mlogloss:0.056286\n",
      "[35]\ttrain-mlogloss:0.046066\ttest-mlogloss:0.052652\n",
      "[36]\ttrain-mlogloss:0.042582\ttest-mlogloss:0.049358\n",
      "[37]\ttrain-mlogloss:0.039432\ttest-mlogloss:0.046351\n",
      "[38]\ttrain-mlogloss:0.036519\ttest-mlogloss:0.043617\n",
      "[39]\ttrain-mlogloss:0.033915\ttest-mlogloss:0.041149\n",
      "[40]\ttrain-mlogloss:0.031525\ttest-mlogloss:0.03896\n",
      "[41]\ttrain-mlogloss:0.029351\ttest-mlogloss:0.03691\n",
      "[42]\ttrain-mlogloss:0.027354\ttest-mlogloss:0.035044\n",
      "[43]\ttrain-mlogloss:0.025501\ttest-mlogloss:0.033393\n",
      "[44]\ttrain-mlogloss:0.023816\ttest-mlogloss:0.031897\n",
      "[45]\ttrain-mlogloss:0.022308\ttest-mlogloss:0.030544\n",
      "[46]\ttrain-mlogloss:0.020892\ttest-mlogloss:0.029308\n",
      "[47]\ttrain-mlogloss:0.019595\ttest-mlogloss:0.028234\n",
      "[48]\ttrain-mlogloss:0.018417\ttest-mlogloss:0.027198\n",
      "[49]\ttrain-mlogloss:0.017332\ttest-mlogloss:0.026287\n",
      "[50]\ttrain-mlogloss:0.016352\ttest-mlogloss:0.02548\n",
      "[51]\ttrain-mlogloss:0.015452\ttest-mlogloss:0.024729\n",
      "[52]\ttrain-mlogloss:0.014638\ttest-mlogloss:0.024068\n",
      "[53]\ttrain-mlogloss:0.013855\ttest-mlogloss:0.023445\n",
      "[54]\ttrain-mlogloss:0.013148\ttest-mlogloss:0.022854\n",
      "[55]\ttrain-mlogloss:0.012488\ttest-mlogloss:0.022377\n",
      "[56]\ttrain-mlogloss:0.011916\ttest-mlogloss:0.021953\n",
      "[57]\ttrain-mlogloss:0.011348\ttest-mlogloss:0.021523\n",
      "[58]\ttrain-mlogloss:0.010836\ttest-mlogloss:0.021144\n",
      "[59]\ttrain-mlogloss:0.01036\ttest-mlogloss:0.020833\n",
      "[60]\ttrain-mlogloss:0.009917\ttest-mlogloss:0.020538\n",
      "[61]\ttrain-mlogloss:0.009491\ttest-mlogloss:0.020213\n",
      "[62]\ttrain-mlogloss:0.009093\ttest-mlogloss:0.019943\n",
      "[63]\ttrain-mlogloss:0.008713\ttest-mlogloss:0.019711\n",
      "[64]\ttrain-mlogloss:0.008392\ttest-mlogloss:0.019496\n",
      "[65]\ttrain-mlogloss:0.00808\ttest-mlogloss:0.019318\n",
      "[66]\ttrain-mlogloss:0.007758\ttest-mlogloss:0.019124\n",
      "[67]\ttrain-mlogloss:0.007466\ttest-mlogloss:0.018955\n",
      "[68]\ttrain-mlogloss:0.007196\ttest-mlogloss:0.018829\n",
      "[69]\ttrain-mlogloss:0.006951\ttest-mlogloss:0.018704\n",
      "[70]\ttrain-mlogloss:0.006714\ttest-mlogloss:0.018559\n",
      "[71]\ttrain-mlogloss:0.006508\ttest-mlogloss:0.018465\n",
      "[72]\ttrain-mlogloss:0.006303\ttest-mlogloss:0.018345\n",
      "[73]\ttrain-mlogloss:0.006121\ttest-mlogloss:0.018249\n",
      "[74]\ttrain-mlogloss:0.005934\ttest-mlogloss:0.01816\n",
      "[75]\ttrain-mlogloss:0.005751\ttest-mlogloss:0.018084\n",
      "[76]\ttrain-mlogloss:0.005598\ttest-mlogloss:0.018041\n",
      "[77]\ttrain-mlogloss:0.00544\ttest-mlogloss:0.017964\n",
      "[78]\ttrain-mlogloss:0.005272\ttest-mlogloss:0.017911\n",
      "[79]\ttrain-mlogloss:0.005109\ttest-mlogloss:0.017875\n",
      "[80]\ttrain-mlogloss:0.004975\ttest-mlogloss:0.017839\n",
      "[81]\ttrain-mlogloss:0.004839\ttest-mlogloss:0.017809\n",
      "[82]\ttrain-mlogloss:0.004725\ttest-mlogloss:0.017794\n",
      "[83]\ttrain-mlogloss:0.004636\ttest-mlogloss:0.01775\n",
      "[84]\ttrain-mlogloss:0.004522\ttest-mlogloss:0.017694\n",
      "[85]\ttrain-mlogloss:0.004419\ttest-mlogloss:0.017663\n",
      "[86]\ttrain-mlogloss:0.004306\ttest-mlogloss:0.017646\n",
      "[87]\ttrain-mlogloss:0.004224\ttest-mlogloss:0.017621\n",
      "[88]\ttrain-mlogloss:0.004127\ttest-mlogloss:0.017592\n",
      "[89]\ttrain-mlogloss:0.004022\ttest-mlogloss:0.017574\n",
      "[90]\ttrain-mlogloss:0.003936\ttest-mlogloss:0.01756\n",
      "[91]\ttrain-mlogloss:0.00386\ttest-mlogloss:0.017564\n",
      "[92]\ttrain-mlogloss:0.003782\ttest-mlogloss:0.017532\n",
      "[93]\ttrain-mlogloss:0.003707\ttest-mlogloss:0.017513\n",
      "[94]\ttrain-mlogloss:0.003643\ttest-mlogloss:0.01751\n",
      "[95]\ttrain-mlogloss:0.003587\ttest-mlogloss:0.01751\n",
      "[96]\ttrain-mlogloss:0.003511\ttest-mlogloss:0.01752\n",
      "[97]\ttrain-mlogloss:0.003451\ttest-mlogloss:0.017511\n",
      "[98]\ttrain-mlogloss:0.003384\ttest-mlogloss:0.017496\n",
      "[99]\ttrain-mlogloss:0.003316\ttest-mlogloss:0.017451\n",
      "[100]\ttrain-mlogloss:0.003248\ttest-mlogloss:0.017456\n",
      "[101]\ttrain-mlogloss:0.003181\ttest-mlogloss:0.01744\n",
      "[102]\ttrain-mlogloss:0.003127\ttest-mlogloss:0.01742\n",
      "[103]\ttrain-mlogloss:0.003075\ttest-mlogloss:0.01744\n",
      "[104]\ttrain-mlogloss:0.003023\ttest-mlogloss:0.017449\n",
      "[105]\ttrain-mlogloss:0.002974\ttest-mlogloss:0.017459\n",
      "[106]\ttrain-mlogloss:0.002928\ttest-mlogloss:0.017479\n",
      "[107]\ttrain-mlogloss:0.002875\ttest-mlogloss:0.017474\n",
      "[108]\ttrain-mlogloss:0.002831\ttest-mlogloss:0.017445\n",
      "[109]\ttrain-mlogloss:0.002777\ttest-mlogloss:0.017432\n",
      "[110]\ttrain-mlogloss:0.002737\ttest-mlogloss:0.01746\n",
      "[111]\ttrain-mlogloss:0.002704\ttest-mlogloss:0.017445\n",
      "[112]\ttrain-mlogloss:0.002659\ttest-mlogloss:0.017454\n",
      "[113]\ttrain-mlogloss:0.00263\ttest-mlogloss:0.017454\n",
      "[114]\ttrain-mlogloss:0.002591\ttest-mlogloss:0.017474\n",
      "[115]\ttrain-mlogloss:0.002555\ttest-mlogloss:0.017514\n",
      "[116]\ttrain-mlogloss:0.00252\ttest-mlogloss:0.017495\n",
      "[117]\ttrain-mlogloss:0.00248\ttest-mlogloss:0.017495\n",
      "[118]\ttrain-mlogloss:0.002444\ttest-mlogloss:0.017496\n",
      "[119]\ttrain-mlogloss:0.002407\ttest-mlogloss:0.017485\n",
      "[120]\ttrain-mlogloss:0.002379\ttest-mlogloss:0.017487\n",
      "[121]\ttrain-mlogloss:0.002349\ttest-mlogloss:0.017489\n",
      "[122]\ttrain-mlogloss:0.002326\ttest-mlogloss:0.017479\n",
      "Stopping. Best iteration:\n",
      "[102]\ttrain-mlogloss:0.003127\ttest-mlogloss:0.01742\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 31045), (116624,))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = sparse.hstack([train_data.iloc[:,:-1],tr_sparse2]).tocsr()\n",
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.50873\ttest-mlogloss:1.50928\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.29926\ttest-mlogloss:1.30009\n",
      "[2]\ttrain-mlogloss:1.1336\ttest-mlogloss:1.13475\n",
      "[3]\ttrain-mlogloss:0.997892\ttest-mlogloss:0.999252\n",
      "[4]\ttrain-mlogloss:0.884058\ttest-mlogloss:0.885687\n",
      "[5]\ttrain-mlogloss:0.786565\ttest-mlogloss:0.78841\n",
      "[6]\ttrain-mlogloss:0.703069\ttest-mlogloss:0.70515\n",
      "[7]\ttrain-mlogloss:0.629733\ttest-mlogloss:0.632049\n",
      "[8]\ttrain-mlogloss:0.565425\ttest-mlogloss:0.56798\n",
      "[9]\ttrain-mlogloss:0.509215\ttest-mlogloss:0.512037\n",
      "[10]\ttrain-mlogloss:0.459003\ttest-mlogloss:0.462001\n",
      "[11]\ttrain-mlogloss:0.414798\ttest-mlogloss:0.418031\n",
      "[12]\ttrain-mlogloss:0.374799\ttest-mlogloss:0.378196\n",
      "[13]\ttrain-mlogloss:0.339603\ttest-mlogloss:0.343184\n",
      "[14]\ttrain-mlogloss:0.307574\ttest-mlogloss:0.311298\n",
      "[15]\ttrain-mlogloss:0.278859\ttest-mlogloss:0.28273\n",
      "[16]\ttrain-mlogloss:0.253093\ttest-mlogloss:0.25714\n",
      "[17]\ttrain-mlogloss:0.22992\ttest-mlogloss:0.234089\n",
      "[18]\ttrain-mlogloss:0.209099\ttest-mlogloss:0.213373\n",
      "[19]\ttrain-mlogloss:0.190321\ttest-mlogloss:0.19469\n",
      "[20]\ttrain-mlogloss:0.17322\ttest-mlogloss:0.17794\n",
      "[21]\ttrain-mlogloss:0.157834\ttest-mlogloss:0.162757\n",
      "[22]\ttrain-mlogloss:0.143958\ttest-mlogloss:0.149061\n",
      "[23]\ttrain-mlogloss:0.131338\ttest-mlogloss:0.136657\n",
      "[24]\ttrain-mlogloss:0.12\ttest-mlogloss:0.125449\n",
      "[25]\ttrain-mlogloss:0.109671\ttest-mlogloss:0.115258\n",
      "[26]\ttrain-mlogloss:0.100371\ttest-mlogloss:0.106185\n",
      "[27]\ttrain-mlogloss:0.092065\ttest-mlogloss:0.098007\n",
      "[28]\ttrain-mlogloss:0.084473\ttest-mlogloss:0.090582\n",
      "[29]\ttrain-mlogloss:0.077615\ttest-mlogloss:0.083945\n",
      "[30]\ttrain-mlogloss:0.071358\ttest-mlogloss:0.077819\n",
      "[31]\ttrain-mlogloss:0.065683\ttest-mlogloss:0.072309\n",
      "[32]\ttrain-mlogloss:0.060478\ttest-mlogloss:0.067304\n",
      "[33]\ttrain-mlogloss:0.055724\ttest-mlogloss:0.062725\n",
      "[34]\ttrain-mlogloss:0.051507\ttest-mlogloss:0.05864\n",
      "[35]\ttrain-mlogloss:0.047673\ttest-mlogloss:0.054996\n",
      "[36]\ttrain-mlogloss:0.044166\ttest-mlogloss:0.051714\n",
      "[37]\ttrain-mlogloss:0.04099\ttest-mlogloss:0.048698\n",
      "[38]\ttrain-mlogloss:0.038035\ttest-mlogloss:0.045928\n",
      "[39]\ttrain-mlogloss:0.035297\ttest-mlogloss:0.043366\n",
      "[40]\ttrain-mlogloss:0.032877\ttest-mlogloss:0.041107\n",
      "[41]\ttrain-mlogloss:0.030641\ttest-mlogloss:0.039012\n",
      "[42]\ttrain-mlogloss:0.028634\ttest-mlogloss:0.037184\n",
      "[43]\ttrain-mlogloss:0.026799\ttest-mlogloss:0.035515\n",
      "[44]\ttrain-mlogloss:0.025111\ttest-mlogloss:0.033991\n",
      "[45]\ttrain-mlogloss:0.023561\ttest-mlogloss:0.032579\n",
      "[46]\ttrain-mlogloss:0.022147\ttest-mlogloss:0.031279\n",
      "[47]\ttrain-mlogloss:0.020782\ttest-mlogloss:0.030068\n",
      "[48]\ttrain-mlogloss:0.019573\ttest-mlogloss:0.029002\n",
      "[49]\ttrain-mlogloss:0.018465\ttest-mlogloss:0.02803\n",
      "[50]\ttrain-mlogloss:0.017535\ttest-mlogloss:0.027216\n",
      "[51]\ttrain-mlogloss:0.016609\ttest-mlogloss:0.026431\n",
      "[52]\ttrain-mlogloss:0.015772\ttest-mlogloss:0.025764\n",
      "[53]\ttrain-mlogloss:0.014958\ttest-mlogloss:0.025094\n",
      "[54]\ttrain-mlogloss:0.014204\ttest-mlogloss:0.024503\n",
      "[55]\ttrain-mlogloss:0.013502\ttest-mlogloss:0.023969\n",
      "[56]\ttrain-mlogloss:0.012844\ttest-mlogloss:0.023467\n",
      "[57]\ttrain-mlogloss:0.012207\ttest-mlogloss:0.022932\n",
      "[58]\ttrain-mlogloss:0.011644\ttest-mlogloss:0.022529\n",
      "[59]\ttrain-mlogloss:0.01116\ttest-mlogloss:0.022146\n",
      "[60]\ttrain-mlogloss:0.010683\ttest-mlogloss:0.02182\n",
      "[61]\ttrain-mlogloss:0.010229\ttest-mlogloss:0.021494\n",
      "[62]\ttrain-mlogloss:0.009821\ttest-mlogloss:0.021231\n",
      "[63]\ttrain-mlogloss:0.009429\ttest-mlogloss:0.020947\n",
      "[64]\ttrain-mlogloss:0.009086\ttest-mlogloss:0.020701\n",
      "[65]\ttrain-mlogloss:0.008751\ttest-mlogloss:0.02049\n",
      "[66]\ttrain-mlogloss:0.008451\ttest-mlogloss:0.020316\n",
      "[67]\ttrain-mlogloss:0.008169\ttest-mlogloss:0.020141\n",
      "[68]\ttrain-mlogloss:0.007891\ttest-mlogloss:0.019969\n",
      "[69]\ttrain-mlogloss:0.007636\ttest-mlogloss:0.019801\n",
      "[70]\ttrain-mlogloss:0.007385\ttest-mlogloss:0.019653\n",
      "[71]\ttrain-mlogloss:0.007171\ttest-mlogloss:0.019561\n",
      "[72]\ttrain-mlogloss:0.006925\ttest-mlogloss:0.019421\n",
      "[73]\ttrain-mlogloss:0.006729\ttest-mlogloss:0.0193\n",
      "[74]\ttrain-mlogloss:0.006543\ttest-mlogloss:0.019207\n",
      "[75]\ttrain-mlogloss:0.006368\ttest-mlogloss:0.019097\n",
      "[76]\ttrain-mlogloss:0.006193\ttest-mlogloss:0.019021\n",
      "[77]\ttrain-mlogloss:0.006035\ttest-mlogloss:0.018962\n",
      "[78]\ttrain-mlogloss:0.005879\ttest-mlogloss:0.018915\n",
      "[79]\ttrain-mlogloss:0.005716\ttest-mlogloss:0.018823\n",
      "[80]\ttrain-mlogloss:0.005569\ttest-mlogloss:0.018735\n",
      "[81]\ttrain-mlogloss:0.005441\ttest-mlogloss:0.018683\n",
      "[82]\ttrain-mlogloss:0.005302\ttest-mlogloss:0.018639\n",
      "[83]\ttrain-mlogloss:0.005167\ttest-mlogloss:0.018588\n",
      "[84]\ttrain-mlogloss:0.005048\ttest-mlogloss:0.01854\n",
      "[85]\ttrain-mlogloss:0.004932\ttest-mlogloss:0.018518\n",
      "[86]\ttrain-mlogloss:0.004825\ttest-mlogloss:0.018476\n",
      "[87]\ttrain-mlogloss:0.004715\ttest-mlogloss:0.018441\n",
      "[88]\ttrain-mlogloss:0.004609\ttest-mlogloss:0.01838\n",
      "[89]\ttrain-mlogloss:0.00451\ttest-mlogloss:0.01834\n",
      "[90]\ttrain-mlogloss:0.004422\ttest-mlogloss:0.018311\n",
      "[91]\ttrain-mlogloss:0.004318\ttest-mlogloss:0.018282\n",
      "[92]\ttrain-mlogloss:0.00424\ttest-mlogloss:0.018267\n",
      "[93]\ttrain-mlogloss:0.004163\ttest-mlogloss:0.018209\n",
      "[94]\ttrain-mlogloss:0.004079\ttest-mlogloss:0.018193\n",
      "[95]\ttrain-mlogloss:0.004015\ttest-mlogloss:0.018203\n",
      "[96]\ttrain-mlogloss:0.00394\ttest-mlogloss:0.018177\n",
      "[97]\ttrain-mlogloss:0.003872\ttest-mlogloss:0.018158\n",
      "[98]\ttrain-mlogloss:0.003789\ttest-mlogloss:0.018139\n",
      "[99]\ttrain-mlogloss:0.003724\ttest-mlogloss:0.018125\n",
      "[100]\ttrain-mlogloss:0.003667\ttest-mlogloss:0.018132\n",
      "[101]\ttrain-mlogloss:0.003599\ttest-mlogloss:0.018119\n",
      "[102]\ttrain-mlogloss:0.00353\ttest-mlogloss:0.018135\n",
      "[103]\ttrain-mlogloss:0.003482\ttest-mlogloss:0.018124\n",
      "[104]\ttrain-mlogloss:0.003435\ttest-mlogloss:0.018115\n",
      "[105]\ttrain-mlogloss:0.003382\ttest-mlogloss:0.018138\n",
      "[106]\ttrain-mlogloss:0.00332\ttest-mlogloss:0.018128\n",
      "[107]\ttrain-mlogloss:0.003257\ttest-mlogloss:0.018143\n",
      "[108]\ttrain-mlogloss:0.003208\ttest-mlogloss:0.018117\n",
      "[109]\ttrain-mlogloss:0.003157\ttest-mlogloss:0.018108\n",
      "[110]\ttrain-mlogloss:0.003118\ttest-mlogloss:0.01811\n",
      "[111]\ttrain-mlogloss:0.003076\ttest-mlogloss:0.018133\n",
      "[112]\ttrain-mlogloss:0.00303\ttest-mlogloss:0.018139\n",
      "[113]\ttrain-mlogloss:0.002983\ttest-mlogloss:0.018155\n",
      "[114]\ttrain-mlogloss:0.002938\ttest-mlogloss:0.018164\n",
      "[115]\ttrain-mlogloss:0.002895\ttest-mlogloss:0.018163\n",
      "[116]\ttrain-mlogloss:0.00286\ttest-mlogloss:0.018174\n",
      "[117]\ttrain-mlogloss:0.002818\ttest-mlogloss:0.018142\n",
      "[118]\ttrain-mlogloss:0.002775\ttest-mlogloss:0.018156\n",
      "[119]\ttrain-mlogloss:0.002742\ttest-mlogloss:0.018147\n",
      "[120]\ttrain-mlogloss:0.002707\ttest-mlogloss:0.018137\n",
      "[121]\ttrain-mlogloss:0.002671\ttest-mlogloss:0.018152\n",
      "[122]\ttrain-mlogloss:0.002636\ttest-mlogloss:0.018137\n",
      "[123]\ttrain-mlogloss:0.0026\ttest-mlogloss:0.018173\n",
      "[124]\ttrain-mlogloss:0.002568\ttest-mlogloss:0.018188\n",
      "[125]\ttrain-mlogloss:0.002532\ttest-mlogloss:0.018188\n",
      "[126]\ttrain-mlogloss:0.002487\ttest-mlogloss:0.018212\n",
      "[127]\ttrain-mlogloss:0.002455\ttest-mlogloss:0.018232\n",
      "[128]\ttrain-mlogloss:0.00242\ttest-mlogloss:0.01824\n",
      "[129]\ttrain-mlogloss:0.002387\ttest-mlogloss:0.018246\n",
      "Stopping. Best iteration:\n",
      "[109]\ttrain-mlogloss:0.003157\ttest-mlogloss:0.018108\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 369), (116624,))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = sparse.hstack([train_data.iloc[:,:-1],tr_sparse3]).tocsr()\n",
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.51206\ttest-mlogloss:1.5126\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.30554\ttest-mlogloss:1.30661\n",
      "[2]\ttrain-mlogloss:1.1425\ttest-mlogloss:1.144\n",
      "[3]\ttrain-mlogloss:1.00876\ttest-mlogloss:1.01049\n",
      "[4]\ttrain-mlogloss:0.896565\ttest-mlogloss:0.89859\n",
      "[5]\ttrain-mlogloss:0.800499\ttest-mlogloss:0.802884\n",
      "[6]\ttrain-mlogloss:0.717618\ttest-mlogloss:0.720431\n",
      "[7]\ttrain-mlogloss:0.645337\ttest-mlogloss:0.648601\n",
      "[8]\ttrain-mlogloss:0.581583\ttest-mlogloss:0.585146\n",
      "[9]\ttrain-mlogloss:0.525202\ttest-mlogloss:0.528974\n",
      "[10]\ttrain-mlogloss:0.475117\ttest-mlogloss:0.47917\n",
      "[11]\ttrain-mlogloss:0.430497\ttest-mlogloss:0.434822\n",
      "[12]\ttrain-mlogloss:0.390687\ttest-mlogloss:0.395261\n",
      "[13]\ttrain-mlogloss:0.355104\ttest-mlogloss:0.359887\n",
      "[14]\ttrain-mlogloss:0.323217\ttest-mlogloss:0.328204\n",
      "[15]\ttrain-mlogloss:0.295079\ttest-mlogloss:0.300329\n",
      "[16]\ttrain-mlogloss:0.269408\ttest-mlogloss:0.274926\n",
      "[17]\ttrain-mlogloss:0.246592\ttest-mlogloss:0.252497\n",
      "[18]\ttrain-mlogloss:0.225747\ttest-mlogloss:0.23199\n",
      "[19]\ttrain-mlogloss:0.206858\ttest-mlogloss:0.213419\n",
      "[20]\ttrain-mlogloss:0.189795\ttest-mlogloss:0.196655\n",
      "[21]\ttrain-mlogloss:0.174556\ttest-mlogloss:0.181674\n",
      "[22]\ttrain-mlogloss:0.160586\ttest-mlogloss:0.167959\n",
      "[23]\ttrain-mlogloss:0.147832\ttest-mlogloss:0.155546\n",
      "[24]\ttrain-mlogloss:0.136389\ttest-mlogloss:0.144339\n",
      "[25]\ttrain-mlogloss:0.126065\ttest-mlogloss:0.13434\n",
      "[26]\ttrain-mlogloss:0.116579\ttest-mlogloss:0.125144\n",
      "[27]\ttrain-mlogloss:0.107626\ttest-mlogloss:0.116521\n",
      "[28]\ttrain-mlogloss:0.099858\ttest-mlogloss:0.108988\n",
      "[29]\ttrain-mlogloss:0.09295\ttest-mlogloss:0.102337\n",
      "[30]\ttrain-mlogloss:0.086384\ttest-mlogloss:0.096003\n",
      "[31]\ttrain-mlogloss:0.080422\ttest-mlogloss:0.090235\n",
      "[32]\ttrain-mlogloss:0.075086\ttest-mlogloss:0.085111\n",
      "[33]\ttrain-mlogloss:0.070131\ttest-mlogloss:0.080343\n",
      "[34]\ttrain-mlogloss:0.065498\ttest-mlogloss:0.076045\n",
      "[35]\ttrain-mlogloss:0.061329\ttest-mlogloss:0.072164\n",
      "[36]\ttrain-mlogloss:0.057649\ttest-mlogloss:0.068732\n",
      "[37]\ttrain-mlogloss:0.054247\ttest-mlogloss:0.065555\n",
      "[38]\ttrain-mlogloss:0.051019\ttest-mlogloss:0.062576\n",
      "[39]\ttrain-mlogloss:0.048055\ttest-mlogloss:0.059853\n",
      "[40]\ttrain-mlogloss:0.04529\ttest-mlogloss:0.057379\n",
      "[41]\ttrain-mlogloss:0.042857\ttest-mlogloss:0.055118\n",
      "[42]\ttrain-mlogloss:0.040548\ttest-mlogloss:0.053067\n",
      "[43]\ttrain-mlogloss:0.038482\ttest-mlogloss:0.051192\n",
      "[44]\ttrain-mlogloss:0.036504\ttest-mlogloss:0.049479\n",
      "[45]\ttrain-mlogloss:0.034761\ttest-mlogloss:0.047891\n",
      "[46]\ttrain-mlogloss:0.033137\ttest-mlogloss:0.046483\n",
      "[47]\ttrain-mlogloss:0.031622\ttest-mlogloss:0.045173\n",
      "[48]\ttrain-mlogloss:0.030202\ttest-mlogloss:0.043911\n",
      "[49]\ttrain-mlogloss:0.028835\ttest-mlogloss:0.042774\n",
      "[50]\ttrain-mlogloss:0.027598\ttest-mlogloss:0.041807\n",
      "[51]\ttrain-mlogloss:0.026483\ttest-mlogloss:0.040892\n",
      "[52]\ttrain-mlogloss:0.025453\ttest-mlogloss:0.040037\n",
      "[53]\ttrain-mlogloss:0.02452\ttest-mlogloss:0.039247\n",
      "[54]\ttrain-mlogloss:0.023738\ttest-mlogloss:0.038624\n",
      "[55]\ttrain-mlogloss:0.022907\ttest-mlogloss:0.037934\n",
      "[56]\ttrain-mlogloss:0.022147\ttest-mlogloss:0.037275\n",
      "[57]\ttrain-mlogloss:0.021443\ttest-mlogloss:0.036758\n",
      "[58]\ttrain-mlogloss:0.020791\ttest-mlogloss:0.036185\n",
      "[59]\ttrain-mlogloss:0.020124\ttest-mlogloss:0.03563\n",
      "[60]\ttrain-mlogloss:0.019478\ttest-mlogloss:0.035185\n",
      "[61]\ttrain-mlogloss:0.018933\ttest-mlogloss:0.034782\n",
      "[62]\ttrain-mlogloss:0.018386\ttest-mlogloss:0.034382\n",
      "[63]\ttrain-mlogloss:0.017869\ttest-mlogloss:0.034029\n",
      "[64]\ttrain-mlogloss:0.017438\ttest-mlogloss:0.033693\n",
      "[65]\ttrain-mlogloss:0.016959\ttest-mlogloss:0.033296\n",
      "[66]\ttrain-mlogloss:0.016539\ttest-mlogloss:0.033001\n",
      "[67]\ttrain-mlogloss:0.016139\ttest-mlogloss:0.032716\n",
      "[68]\ttrain-mlogloss:0.015706\ttest-mlogloss:0.032428\n",
      "[69]\ttrain-mlogloss:0.015268\ttest-mlogloss:0.03217\n",
      "[70]\ttrain-mlogloss:0.014908\ttest-mlogloss:0.031961\n",
      "[71]\ttrain-mlogloss:0.014556\ttest-mlogloss:0.031726\n",
      "[72]\ttrain-mlogloss:0.01419\ttest-mlogloss:0.031549\n",
      "[73]\ttrain-mlogloss:0.013904\ttest-mlogloss:0.031322\n",
      "[74]\ttrain-mlogloss:0.013608\ttest-mlogloss:0.031149\n",
      "[75]\ttrain-mlogloss:0.013274\ttest-mlogloss:0.030977\n",
      "[76]\ttrain-mlogloss:0.012991\ttest-mlogloss:0.030804\n",
      "[77]\ttrain-mlogloss:0.012706\ttest-mlogloss:0.030663\n",
      "[78]\ttrain-mlogloss:0.012419\ttest-mlogloss:0.030492\n",
      "[79]\ttrain-mlogloss:0.012088\ttest-mlogloss:0.030302\n",
      "[80]\ttrain-mlogloss:0.01188\ttest-mlogloss:0.030212\n",
      "[81]\ttrain-mlogloss:0.011687\ttest-mlogloss:0.030161\n",
      "[82]\ttrain-mlogloss:0.01148\ttest-mlogloss:0.030071\n",
      "[83]\ttrain-mlogloss:0.011306\ttest-mlogloss:0.029982\n",
      "[84]\ttrain-mlogloss:0.011114\ttest-mlogloss:0.029878\n",
      "[85]\ttrain-mlogloss:0.010916\ttest-mlogloss:0.029781\n",
      "[86]\ttrain-mlogloss:0.010723\ttest-mlogloss:0.029724\n",
      "[87]\ttrain-mlogloss:0.010543\ttest-mlogloss:0.029671\n",
      "[88]\ttrain-mlogloss:0.010311\ttest-mlogloss:0.029576\n",
      "[89]\ttrain-mlogloss:0.010134\ttest-mlogloss:0.029496\n",
      "[90]\ttrain-mlogloss:0.00994\ttest-mlogloss:0.029453\n",
      "[91]\ttrain-mlogloss:0.009717\ttest-mlogloss:0.029395\n",
      "[92]\ttrain-mlogloss:0.009563\ttest-mlogloss:0.029294\n",
      "[93]\ttrain-mlogloss:0.009411\ttest-mlogloss:0.029283\n",
      "[94]\ttrain-mlogloss:0.009269\ttest-mlogloss:0.029226\n",
      "[95]\ttrain-mlogloss:0.009148\ttest-mlogloss:0.029172\n",
      "[96]\ttrain-mlogloss:0.008983\ttest-mlogloss:0.029096\n",
      "[97]\ttrain-mlogloss:0.008822\ttest-mlogloss:0.029042\n",
      "[98]\ttrain-mlogloss:0.00868\ttest-mlogloss:0.029007\n",
      "[99]\ttrain-mlogloss:0.008516\ttest-mlogloss:0.028952\n",
      "[100]\ttrain-mlogloss:0.008389\ttest-mlogloss:0.028903\n",
      "[101]\ttrain-mlogloss:0.008273\ttest-mlogloss:0.028842\n",
      "[102]\ttrain-mlogloss:0.00818\ttest-mlogloss:0.028805\n",
      "[103]\ttrain-mlogloss:0.008081\ttest-mlogloss:0.028749\n",
      "[104]\ttrain-mlogloss:0.007977\ttest-mlogloss:0.028708\n",
      "[105]\ttrain-mlogloss:0.007859\ttest-mlogloss:0.028695\n",
      "[106]\ttrain-mlogloss:0.007705\ttest-mlogloss:0.028641\n",
      "[107]\ttrain-mlogloss:0.007579\ttest-mlogloss:0.028599\n",
      "[108]\ttrain-mlogloss:0.007485\ttest-mlogloss:0.028589\n",
      "[109]\ttrain-mlogloss:0.007372\ttest-mlogloss:0.028581\n",
      "[110]\ttrain-mlogloss:0.00727\ttest-mlogloss:0.028531\n",
      "[111]\ttrain-mlogloss:0.007125\ttest-mlogloss:0.028519\n",
      "[112]\ttrain-mlogloss:0.007031\ttest-mlogloss:0.028509\n",
      "[113]\ttrain-mlogloss:0.006925\ttest-mlogloss:0.028512\n",
      "[114]\ttrain-mlogloss:0.006826\ttest-mlogloss:0.028496\n",
      "[115]\ttrain-mlogloss:0.006753\ttest-mlogloss:0.028438\n",
      "[116]\ttrain-mlogloss:0.00667\ttest-mlogloss:0.028428\n",
      "[117]\ttrain-mlogloss:0.006579\ttest-mlogloss:0.028385\n",
      "[118]\ttrain-mlogloss:0.006508\ttest-mlogloss:0.028342\n",
      "[119]\ttrain-mlogloss:0.006426\ttest-mlogloss:0.028302\n",
      "[120]\ttrain-mlogloss:0.006366\ttest-mlogloss:0.028285\n",
      "[121]\ttrain-mlogloss:0.006294\ttest-mlogloss:0.028325\n",
      "[122]\ttrain-mlogloss:0.006226\ttest-mlogloss:0.028289\n",
      "[123]\ttrain-mlogloss:0.006155\ttest-mlogloss:0.028254\n",
      "[124]\ttrain-mlogloss:0.006064\ttest-mlogloss:0.028228\n",
      "[125]\ttrain-mlogloss:0.005984\ttest-mlogloss:0.02821\n",
      "[126]\ttrain-mlogloss:0.005899\ttest-mlogloss:0.028215\n",
      "[127]\ttrain-mlogloss:0.005827\ttest-mlogloss:0.028248\n",
      "[128]\ttrain-mlogloss:0.005768\ttest-mlogloss:0.028224\n",
      "[129]\ttrain-mlogloss:0.005707\ttest-mlogloss:0.028206\n",
      "[130]\ttrain-mlogloss:0.005623\ttest-mlogloss:0.028165\n",
      "[131]\ttrain-mlogloss:0.005555\ttest-mlogloss:0.028165\n",
      "[132]\ttrain-mlogloss:0.005483\ttest-mlogloss:0.028206\n",
      "[133]\ttrain-mlogloss:0.005428\ttest-mlogloss:0.028185\n",
      "[134]\ttrain-mlogloss:0.005376\ttest-mlogloss:0.028168\n",
      "[135]\ttrain-mlogloss:0.005313\ttest-mlogloss:0.028156\n",
      "[136]\ttrain-mlogloss:0.005262\ttest-mlogloss:0.028152\n",
      "[137]\ttrain-mlogloss:0.005193\ttest-mlogloss:0.028135\n",
      "[138]\ttrain-mlogloss:0.005129\ttest-mlogloss:0.028107\n",
      "[139]\ttrain-mlogloss:0.005068\ttest-mlogloss:0.028114\n",
      "[140]\ttrain-mlogloss:0.005009\ttest-mlogloss:0.028077\n",
      "[141]\ttrain-mlogloss:0.00497\ttest-mlogloss:0.028043\n",
      "[142]\ttrain-mlogloss:0.00491\ttest-mlogloss:0.028057\n",
      "[143]\ttrain-mlogloss:0.004851\ttest-mlogloss:0.028045\n",
      "[144]\ttrain-mlogloss:0.00479\ttest-mlogloss:0.028026\n",
      "[145]\ttrain-mlogloss:0.004729\ttest-mlogloss:0.028044\n",
      "[146]\ttrain-mlogloss:0.004686\ttest-mlogloss:0.028065\n",
      "[147]\ttrain-mlogloss:0.004615\ttest-mlogloss:0.028072\n",
      "[148]\ttrain-mlogloss:0.004564\ttest-mlogloss:0.028097\n",
      "[149]\ttrain-mlogloss:0.004492\ttest-mlogloss:0.028106\n",
      "[150]\ttrain-mlogloss:0.004424\ttest-mlogloss:0.028111\n",
      "[151]\ttrain-mlogloss:0.004389\ttest-mlogloss:0.028099\n",
      "[152]\ttrain-mlogloss:0.004346\ttest-mlogloss:0.028051\n",
      "[153]\ttrain-mlogloss:0.004318\ttest-mlogloss:0.028012\n",
      "[154]\ttrain-mlogloss:0.004285\ttest-mlogloss:0.027992\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[155]\ttrain-mlogloss:0.004248\ttest-mlogloss:0.027998\n",
      "[156]\ttrain-mlogloss:0.004219\ttest-mlogloss:0.028004\n",
      "[157]\ttrain-mlogloss:0.004176\ttest-mlogloss:0.027986\n",
      "[158]\ttrain-mlogloss:0.004129\ttest-mlogloss:0.027994\n",
      "[159]\ttrain-mlogloss:0.004098\ttest-mlogloss:0.028004\n",
      "[160]\ttrain-mlogloss:0.004059\ttest-mlogloss:0.027993\n",
      "[161]\ttrain-mlogloss:0.004022\ttest-mlogloss:0.028005\n",
      "[162]\ttrain-mlogloss:0.003983\ttest-mlogloss:0.028024\n",
      "[163]\ttrain-mlogloss:0.003945\ttest-mlogloss:0.028044\n",
      "[164]\ttrain-mlogloss:0.003895\ttest-mlogloss:0.028062\n",
      "[165]\ttrain-mlogloss:0.00385\ttest-mlogloss:0.02808\n",
      "[166]\ttrain-mlogloss:0.003809\ttest-mlogloss:0.02809\n",
      "[167]\ttrain-mlogloss:0.003764\ttest-mlogloss:0.028071\n",
      "[168]\ttrain-mlogloss:0.003721\ttest-mlogloss:0.028091\n",
      "[169]\ttrain-mlogloss:0.003687\ttest-mlogloss:0.028099\n",
      "[170]\ttrain-mlogloss:0.003647\ttest-mlogloss:0.028089\n",
      "[171]\ttrain-mlogloss:0.00361\ttest-mlogloss:0.028099\n",
      "[172]\ttrain-mlogloss:0.003578\ttest-mlogloss:0.028089\n",
      "[173]\ttrain-mlogloss:0.00355\ttest-mlogloss:0.028071\n",
      "[174]\ttrain-mlogloss:0.003522\ttest-mlogloss:0.028066\n",
      "[175]\ttrain-mlogloss:0.003487\ttest-mlogloss:0.028064\n",
      "[176]\ttrain-mlogloss:0.003452\ttest-mlogloss:0.028076\n",
      "[177]\ttrain-mlogloss:0.00342\ttest-mlogloss:0.028098\n",
      "Stopping. Best iteration:\n",
      "[157]\ttrain-mlogloss:0.004176\ttest-mlogloss:0.027986\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 31292), (116624,))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = sparse.hstack([tr_sparse2,tr_sparse3]).tocsr()\n",
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.50802\ttest-mlogloss:1.50853\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.29872\ttest-mlogloss:1.29963\n",
      "[2]\ttrain-mlogloss:1.13356\ttest-mlogloss:1.13484\n",
      "[3]\ttrain-mlogloss:0.998775\ttest-mlogloss:1.00031\n",
      "[4]\ttrain-mlogloss:0.885518\ttest-mlogloss:0.887339\n",
      "[5]\ttrain-mlogloss:0.788086\ttest-mlogloss:0.790138\n",
      "[6]\ttrain-mlogloss:0.703927\ttest-mlogloss:0.706178\n",
      "[7]\ttrain-mlogloss:0.630999\ttest-mlogloss:0.633506\n",
      "[8]\ttrain-mlogloss:0.56659\ttest-mlogloss:0.569283\n",
      "[9]\ttrain-mlogloss:0.50995\ttest-mlogloss:0.512797\n",
      "[10]\ttrain-mlogloss:0.459784\ttest-mlogloss:0.462826\n",
      "[11]\ttrain-mlogloss:0.415062\ttest-mlogloss:0.418248\n",
      "[12]\ttrain-mlogloss:0.375477\ttest-mlogloss:0.378901\n",
      "[13]\ttrain-mlogloss:0.3398\ttest-mlogloss:0.343425\n",
      "[14]\ttrain-mlogloss:0.307985\ttest-mlogloss:0.311742\n",
      "[15]\ttrain-mlogloss:0.279378\ttest-mlogloss:0.283228\n",
      "[16]\ttrain-mlogloss:0.254066\ttest-mlogloss:0.258069\n",
      "[17]\ttrain-mlogloss:0.23088\ttest-mlogloss:0.234993\n",
      "[18]\ttrain-mlogloss:0.210549\ttest-mlogloss:0.214865\n",
      "[19]\ttrain-mlogloss:0.192001\ttest-mlogloss:0.196466\n",
      "[20]\ttrain-mlogloss:0.175095\ttest-mlogloss:0.179786\n",
      "[21]\ttrain-mlogloss:0.159705\ttest-mlogloss:0.16449\n",
      "[22]\ttrain-mlogloss:0.146213\ttest-mlogloss:0.151214\n",
      "[23]\ttrain-mlogloss:0.133814\ttest-mlogloss:0.139028\n",
      "[24]\ttrain-mlogloss:0.122631\ttest-mlogloss:0.128023\n",
      "[25]\ttrain-mlogloss:0.112344\ttest-mlogloss:0.117792\n",
      "[26]\ttrain-mlogloss:0.103022\ttest-mlogloss:0.108537\n",
      "[27]\ttrain-mlogloss:0.094557\ttest-mlogloss:0.10018\n",
      "[28]\ttrain-mlogloss:0.086908\ttest-mlogloss:0.092691\n",
      "[29]\ttrain-mlogloss:0.080165\ttest-mlogloss:0.086109\n",
      "[30]\ttrain-mlogloss:0.074025\ttest-mlogloss:0.080096\n",
      "[31]\ttrain-mlogloss:0.06838\ttest-mlogloss:0.074549\n",
      "[32]\ttrain-mlogloss:0.063329\ttest-mlogloss:0.069608\n",
      "[33]\ttrain-mlogloss:0.058762\ttest-mlogloss:0.065203\n",
      "[34]\ttrain-mlogloss:0.054579\ttest-mlogloss:0.06119\n",
      "[35]\ttrain-mlogloss:0.050656\ttest-mlogloss:0.057409\n",
      "[36]\ttrain-mlogloss:0.04719\ttest-mlogloss:0.054066\n",
      "[37]\ttrain-mlogloss:0.043997\ttest-mlogloss:0.051098\n",
      "[38]\ttrain-mlogloss:0.04106\ttest-mlogloss:0.048233\n",
      "[39]\ttrain-mlogloss:0.038338\ttest-mlogloss:0.045674\n",
      "[40]\ttrain-mlogloss:0.035833\ttest-mlogloss:0.0433\n",
      "[41]\ttrain-mlogloss:0.033612\ttest-mlogloss:0.041219\n",
      "[42]\ttrain-mlogloss:0.031596\ttest-mlogloss:0.039373\n",
      "[43]\ttrain-mlogloss:0.029668\ttest-mlogloss:0.037629\n",
      "[44]\ttrain-mlogloss:0.027917\ttest-mlogloss:0.036042\n",
      "[45]\ttrain-mlogloss:0.02639\ttest-mlogloss:0.034638\n",
      "[46]\ttrain-mlogloss:0.024903\ttest-mlogloss:0.03329\n",
      "[47]\ttrain-mlogloss:0.023569\ttest-mlogloss:0.032076\n",
      "[48]\ttrain-mlogloss:0.022299\ttest-mlogloss:0.030943\n",
      "[49]\ttrain-mlogloss:0.021142\ttest-mlogloss:0.029947\n",
      "[50]\ttrain-mlogloss:0.020042\ttest-mlogloss:0.02896\n",
      "[51]\ttrain-mlogloss:0.019017\ttest-mlogloss:0.028047\n",
      "[52]\ttrain-mlogloss:0.018162\ttest-mlogloss:0.027307\n",
      "[53]\ttrain-mlogloss:0.017311\ttest-mlogloss:0.026608\n",
      "[54]\ttrain-mlogloss:0.016532\ttest-mlogloss:0.025988\n",
      "[55]\ttrain-mlogloss:0.015804\ttest-mlogloss:0.025426\n",
      "[56]\ttrain-mlogloss:0.015124\ttest-mlogloss:0.024859\n",
      "[57]\ttrain-mlogloss:0.014501\ttest-mlogloss:0.024366\n",
      "[58]\ttrain-mlogloss:0.013874\ttest-mlogloss:0.023882\n",
      "[59]\ttrain-mlogloss:0.013332\ttest-mlogloss:0.023444\n",
      "[60]\ttrain-mlogloss:0.012803\ttest-mlogloss:0.023064\n",
      "[61]\ttrain-mlogloss:0.01232\ttest-mlogloss:0.022659\n",
      "[62]\ttrain-mlogloss:0.011857\ttest-mlogloss:0.022322\n",
      "[63]\ttrain-mlogloss:0.011427\ttest-mlogloss:0.022041\n",
      "[64]\ttrain-mlogloss:0.011016\ttest-mlogloss:0.021787\n",
      "[65]\ttrain-mlogloss:0.010624\ttest-mlogloss:0.021574\n",
      "[66]\ttrain-mlogloss:0.010269\ttest-mlogloss:0.021325\n",
      "[67]\ttrain-mlogloss:0.009882\ttest-mlogloss:0.021098\n",
      "[68]\ttrain-mlogloss:0.009582\ttest-mlogloss:0.020938\n",
      "[69]\ttrain-mlogloss:0.009272\ttest-mlogloss:0.02076\n",
      "[70]\ttrain-mlogloss:0.008989\ttest-mlogloss:0.020584\n",
      "[71]\ttrain-mlogloss:0.008753\ttest-mlogloss:0.02045\n",
      "[72]\ttrain-mlogloss:0.008536\ttest-mlogloss:0.020346\n",
      "[73]\ttrain-mlogloss:0.008312\ttest-mlogloss:0.020219\n",
      "[74]\ttrain-mlogloss:0.008126\ttest-mlogloss:0.020144\n",
      "[75]\ttrain-mlogloss:0.00789\ttest-mlogloss:0.019984\n",
      "[76]\ttrain-mlogloss:0.007656\ttest-mlogloss:0.019864\n",
      "[77]\ttrain-mlogloss:0.007455\ttest-mlogloss:0.019792\n",
      "[78]\ttrain-mlogloss:0.007255\ttest-mlogloss:0.019739\n",
      "[79]\ttrain-mlogloss:0.007077\ttest-mlogloss:0.019656\n",
      "[80]\ttrain-mlogloss:0.006944\ttest-mlogloss:0.019589\n",
      "[81]\ttrain-mlogloss:0.006781\ttest-mlogloss:0.019518\n",
      "[82]\ttrain-mlogloss:0.006652\ttest-mlogloss:0.019474\n",
      "[83]\ttrain-mlogloss:0.006508\ttest-mlogloss:0.019387\n",
      "[84]\ttrain-mlogloss:0.00635\ttest-mlogloss:0.019345\n",
      "[85]\ttrain-mlogloss:0.006226\ttest-mlogloss:0.019309\n",
      "[86]\ttrain-mlogloss:0.0061\ttest-mlogloss:0.019249\n",
      "[87]\ttrain-mlogloss:0.005978\ttest-mlogloss:0.019213\n",
      "[88]\ttrain-mlogloss:0.005864\ttest-mlogloss:0.019151\n",
      "[89]\ttrain-mlogloss:0.005773\ttest-mlogloss:0.019104\n",
      "[90]\ttrain-mlogloss:0.005654\ttest-mlogloss:0.019066\n",
      "[91]\ttrain-mlogloss:0.005529\ttest-mlogloss:0.019028\n",
      "[92]\ttrain-mlogloss:0.005443\ttest-mlogloss:0.018977\n",
      "[93]\ttrain-mlogloss:0.005327\ttest-mlogloss:0.018943\n",
      "[94]\ttrain-mlogloss:0.005227\ttest-mlogloss:0.018911\n",
      "[95]\ttrain-mlogloss:0.005127\ttest-mlogloss:0.018858\n",
      "[96]\ttrain-mlogloss:0.005034\ttest-mlogloss:0.018837\n",
      "[97]\ttrain-mlogloss:0.004932\ttest-mlogloss:0.018805\n",
      "[98]\ttrain-mlogloss:0.004848\ttest-mlogloss:0.018784\n",
      "[99]\ttrain-mlogloss:0.004776\ttest-mlogloss:0.018773\n",
      "[100]\ttrain-mlogloss:0.00468\ttest-mlogloss:0.018733\n",
      "[101]\ttrain-mlogloss:0.004609\ttest-mlogloss:0.018711\n",
      "[102]\ttrain-mlogloss:0.00454\ttest-mlogloss:0.018686\n",
      "[103]\ttrain-mlogloss:0.004477\ttest-mlogloss:0.018644\n",
      "[104]\ttrain-mlogloss:0.004387\ttest-mlogloss:0.018643\n",
      "[105]\ttrain-mlogloss:0.00433\ttest-mlogloss:0.018633\n",
      "[106]\ttrain-mlogloss:0.004258\ttest-mlogloss:0.018673\n",
      "[107]\ttrain-mlogloss:0.004212\ttest-mlogloss:0.018649\n",
      "[108]\ttrain-mlogloss:0.004144\ttest-mlogloss:0.018645\n",
      "[109]\ttrain-mlogloss:0.004086\ttest-mlogloss:0.018639\n",
      "[110]\ttrain-mlogloss:0.00403\ttest-mlogloss:0.018638\n",
      "[111]\ttrain-mlogloss:0.00397\ttest-mlogloss:0.018611\n",
      "[112]\ttrain-mlogloss:0.003903\ttest-mlogloss:0.018585\n",
      "[113]\ttrain-mlogloss:0.003858\ttest-mlogloss:0.018573\n",
      "[114]\ttrain-mlogloss:0.00382\ttest-mlogloss:0.018579\n",
      "[115]\ttrain-mlogloss:0.003775\ttest-mlogloss:0.018577\n",
      "[116]\ttrain-mlogloss:0.00373\ttest-mlogloss:0.018593\n",
      "[117]\ttrain-mlogloss:0.003698\ttest-mlogloss:0.018561\n",
      "[118]\ttrain-mlogloss:0.003659\ttest-mlogloss:0.01856\n",
      "[119]\ttrain-mlogloss:0.003616\ttest-mlogloss:0.018523\n",
      "[120]\ttrain-mlogloss:0.003572\ttest-mlogloss:0.018537\n",
      "[121]\ttrain-mlogloss:0.003529\ttest-mlogloss:0.018531\n",
      "[122]\ttrain-mlogloss:0.003488\ttest-mlogloss:0.018528\n",
      "[123]\ttrain-mlogloss:0.003443\ttest-mlogloss:0.018522\n",
      "[124]\ttrain-mlogloss:0.003404\ttest-mlogloss:0.01854\n",
      "[125]\ttrain-mlogloss:0.003366\ttest-mlogloss:0.018527\n",
      "[126]\ttrain-mlogloss:0.003321\ttest-mlogloss:0.018525\n",
      "[127]\ttrain-mlogloss:0.003274\ttest-mlogloss:0.018532\n",
      "[128]\ttrain-mlogloss:0.003243\ttest-mlogloss:0.018528\n",
      "[129]\ttrain-mlogloss:0.003204\ttest-mlogloss:0.01855\n",
      "[130]\ttrain-mlogloss:0.003174\ttest-mlogloss:0.018549\n",
      "[131]\ttrain-mlogloss:0.003143\ttest-mlogloss:0.018558\n",
      "[132]\ttrain-mlogloss:0.003115\ttest-mlogloss:0.018552\n",
      "[133]\ttrain-mlogloss:0.003079\ttest-mlogloss:0.018573\n",
      "[134]\ttrain-mlogloss:0.003044\ttest-mlogloss:0.018583\n",
      "[135]\ttrain-mlogloss:0.003015\ttest-mlogloss:0.018594\n",
      "[136]\ttrain-mlogloss:0.002978\ttest-mlogloss:0.018584\n",
      "[137]\ttrain-mlogloss:0.002953\ttest-mlogloss:0.018593\n",
      "[138]\ttrain-mlogloss:0.002914\ttest-mlogloss:0.01859\n",
      "[139]\ttrain-mlogloss:0.002888\ttest-mlogloss:0.018596\n",
      "[140]\ttrain-mlogloss:0.002861\ttest-mlogloss:0.018608\n",
      "[141]\ttrain-mlogloss:0.002836\ttest-mlogloss:0.018651\n",
      "[142]\ttrain-mlogloss:0.002803\ttest-mlogloss:0.018664\n",
      "[143]\ttrain-mlogloss:0.002781\ttest-mlogloss:0.018664\n",
      "Stopping. Best iteration:\n",
      "[123]\ttrain-mlogloss:0.003443\ttest-mlogloss:0.018522\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 97832), (116624,))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = sparse.hstack([tr_sparse1,tr_sparse2,tr_sparse3]).tocsr()\n",
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.50795\ttest-mlogloss:1.50848\n",
      "Multiple eval metrics have been passed: 'test-mlogloss' will be used for early stopping.\n",
      "\n",
      "Will train until test-mlogloss hasn't improved in 20 rounds.\n",
      "[1]\ttrain-mlogloss:1.29805\ttest-mlogloss:1.29886\n",
      "[2]\ttrain-mlogloss:1.1319\ttest-mlogloss:1.1329\n",
      "[3]\ttrain-mlogloss:0.99606\ttest-mlogloss:0.997207\n",
      "[4]\ttrain-mlogloss:0.882086\ttest-mlogloss:0.883416\n",
      "[5]\ttrain-mlogloss:0.78464\ttest-mlogloss:0.786084\n",
      "[6]\ttrain-mlogloss:0.700412\ttest-mlogloss:0.701961\n",
      "[7]\ttrain-mlogloss:0.627189\ttest-mlogloss:0.62893\n",
      "[8]\ttrain-mlogloss:0.562819\ttest-mlogloss:0.564643\n",
      "[9]\ttrain-mlogloss:0.506157\ttest-mlogloss:0.50806\n",
      "[10]\ttrain-mlogloss:0.455973\ttest-mlogloss:0.457956\n",
      "[11]\ttrain-mlogloss:0.411391\ttest-mlogloss:0.413482\n",
      "[12]\ttrain-mlogloss:0.371699\ttest-mlogloss:0.373984\n",
      "[13]\ttrain-mlogloss:0.336233\ttest-mlogloss:0.338705\n",
      "[14]\ttrain-mlogloss:0.304453\ttest-mlogloss:0.307116\n",
      "[15]\ttrain-mlogloss:0.275939\ttest-mlogloss:0.278714\n",
      "[16]\ttrain-mlogloss:0.250372\ttest-mlogloss:0.253327\n",
      "[17]\ttrain-mlogloss:0.22743\ttest-mlogloss:0.230495\n",
      "[18]\ttrain-mlogloss:0.20679\ttest-mlogloss:0.209976\n",
      "[19]\ttrain-mlogloss:0.188107\ttest-mlogloss:0.191397\n",
      "[20]\ttrain-mlogloss:0.171283\ttest-mlogloss:0.174756\n",
      "[21]\ttrain-mlogloss:0.156137\ttest-mlogloss:0.159805\n",
      "[22]\ttrain-mlogloss:0.142475\ttest-mlogloss:0.146262\n",
      "[23]\ttrain-mlogloss:0.130115\ttest-mlogloss:0.134077\n",
      "[24]\ttrain-mlogloss:0.118989\ttest-mlogloss:0.123062\n",
      "[25]\ttrain-mlogloss:0.108882\ttest-mlogloss:0.11316\n",
      "[26]\ttrain-mlogloss:0.0998\ttest-mlogloss:0.104268\n",
      "[27]\ttrain-mlogloss:0.091576\ttest-mlogloss:0.09618\n",
      "[28]\ttrain-mlogloss:0.084115\ttest-mlogloss:0.088933\n",
      "[29]\ttrain-mlogloss:0.077384\ttest-mlogloss:0.082366\n",
      "[30]\ttrain-mlogloss:0.071242\ttest-mlogloss:0.076382\n",
      "[31]\ttrain-mlogloss:0.065675\ttest-mlogloss:0.070944\n",
      "[32]\ttrain-mlogloss:0.060667\ttest-mlogloss:0.066134\n",
      "[33]\ttrain-mlogloss:0.056082\ttest-mlogloss:0.061716\n",
      "[34]\ttrain-mlogloss:0.05191\ttest-mlogloss:0.057677\n",
      "[35]\ttrain-mlogloss:0.048176\ttest-mlogloss:0.054101\n",
      "[36]\ttrain-mlogloss:0.04474\ttest-mlogloss:0.050784\n",
      "[37]\ttrain-mlogloss:0.041598\ttest-mlogloss:0.047873\n",
      "[38]\ttrain-mlogloss:0.038729\ttest-mlogloss:0.045196\n",
      "[39]\ttrain-mlogloss:0.036129\ttest-mlogloss:0.042762\n",
      "[40]\ttrain-mlogloss:0.03371\ttest-mlogloss:0.040533\n",
      "[41]\ttrain-mlogloss:0.03151\ttest-mlogloss:0.038474\n",
      "[42]\ttrain-mlogloss:0.02951\ttest-mlogloss:0.036624\n",
      "[43]\ttrain-mlogloss:0.02767\ttest-mlogloss:0.034945\n",
      "[44]\ttrain-mlogloss:0.02601\ttest-mlogloss:0.033459\n",
      "[45]\ttrain-mlogloss:0.02449\ttest-mlogloss:0.032028\n",
      "[46]\ttrain-mlogloss:0.023071\ttest-mlogloss:0.030759\n",
      "[47]\ttrain-mlogloss:0.021798\ttest-mlogloss:0.029637\n",
      "[48]\ttrain-mlogloss:0.020587\ttest-mlogloss:0.028586\n",
      "[49]\ttrain-mlogloss:0.01949\ttest-mlogloss:0.02766\n",
      "[50]\ttrain-mlogloss:0.018397\ttest-mlogloss:0.026803\n",
      "[51]\ttrain-mlogloss:0.017362\ttest-mlogloss:0.025931\n",
      "[52]\ttrain-mlogloss:0.016461\ttest-mlogloss:0.025186\n",
      "[53]\ttrain-mlogloss:0.015667\ttest-mlogloss:0.024554\n",
      "[54]\ttrain-mlogloss:0.014898\ttest-mlogloss:0.023942\n",
      "[55]\ttrain-mlogloss:0.014184\ttest-mlogloss:0.023388\n",
      "[56]\ttrain-mlogloss:0.013509\ttest-mlogloss:0.022857\n",
      "[57]\ttrain-mlogloss:0.012908\ttest-mlogloss:0.022424\n",
      "[58]\ttrain-mlogloss:0.012341\ttest-mlogloss:0.022011\n",
      "[59]\ttrain-mlogloss:0.011791\ttest-mlogloss:0.021609\n",
      "[60]\ttrain-mlogloss:0.011298\ttest-mlogloss:0.021211\n",
      "[61]\ttrain-mlogloss:0.010788\ttest-mlogloss:0.020913\n",
      "[62]\ttrain-mlogloss:0.010374\ttest-mlogloss:0.020592\n",
      "[63]\ttrain-mlogloss:0.009971\ttest-mlogloss:0.020342\n",
      "[64]\ttrain-mlogloss:0.009587\ttest-mlogloss:0.020122\n",
      "[65]\ttrain-mlogloss:0.009233\ttest-mlogloss:0.019933\n",
      "[66]\ttrain-mlogloss:0.008905\ttest-mlogloss:0.019727\n",
      "[67]\ttrain-mlogloss:0.008593\ttest-mlogloss:0.019526\n",
      "[68]\ttrain-mlogloss:0.008302\ttest-mlogloss:0.019342\n",
      "[69]\ttrain-mlogloss:0.008026\ttest-mlogloss:0.01917\n",
      "[70]\ttrain-mlogloss:0.007787\ttest-mlogloss:0.019035\n",
      "[71]\ttrain-mlogloss:0.007547\ttest-mlogloss:0.01891\n",
      "[72]\ttrain-mlogloss:0.007316\ttest-mlogloss:0.018811\n",
      "[73]\ttrain-mlogloss:0.007109\ttest-mlogloss:0.018715\n",
      "[74]\ttrain-mlogloss:0.006884\ttest-mlogloss:0.018593\n",
      "[75]\ttrain-mlogloss:0.006705\ttest-mlogloss:0.01853\n",
      "[76]\ttrain-mlogloss:0.006561\ttest-mlogloss:0.018453\n",
      "[77]\ttrain-mlogloss:0.006394\ttest-mlogloss:0.018388\n",
      "[78]\ttrain-mlogloss:0.006216\ttest-mlogloss:0.018319\n",
      "[79]\ttrain-mlogloss:0.006063\ttest-mlogloss:0.01824\n",
      "[80]\ttrain-mlogloss:0.005913\ttest-mlogloss:0.018197\n",
      "[81]\ttrain-mlogloss:0.005753\ttest-mlogloss:0.018169\n",
      "[82]\ttrain-mlogloss:0.005616\ttest-mlogloss:0.018153\n",
      "[83]\ttrain-mlogloss:0.00548\ttest-mlogloss:0.018094\n",
      "[84]\ttrain-mlogloss:0.005372\ttest-mlogloss:0.018061\n",
      "[85]\ttrain-mlogloss:0.005241\ttest-mlogloss:0.018021\n",
      "[86]\ttrain-mlogloss:0.005131\ttest-mlogloss:0.017992\n",
      "[87]\ttrain-mlogloss:0.005014\ttest-mlogloss:0.017955\n",
      "[88]\ttrain-mlogloss:0.004893\ttest-mlogloss:0.017932\n",
      "[89]\ttrain-mlogloss:0.004804\ttest-mlogloss:0.017884\n",
      "[90]\ttrain-mlogloss:0.004704\ttest-mlogloss:0.017854\n",
      "[91]\ttrain-mlogloss:0.004607\ttest-mlogloss:0.017815\n",
      "[92]\ttrain-mlogloss:0.004522\ttest-mlogloss:0.017801\n",
      "[93]\ttrain-mlogloss:0.004448\ttest-mlogloss:0.017749\n",
      "[94]\ttrain-mlogloss:0.004377\ttest-mlogloss:0.017741\n",
      "[95]\ttrain-mlogloss:0.004296\ttest-mlogloss:0.017728\n",
      "[96]\ttrain-mlogloss:0.004228\ttest-mlogloss:0.017752\n",
      "[97]\ttrain-mlogloss:0.004131\ttest-mlogloss:0.01776\n",
      "[98]\ttrain-mlogloss:0.004053\ttest-mlogloss:0.017752\n",
      "[99]\ttrain-mlogloss:0.003984\ttest-mlogloss:0.017743\n",
      "[100]\ttrain-mlogloss:0.003917\ttest-mlogloss:0.01772\n",
      "[101]\ttrain-mlogloss:0.003855\ttest-mlogloss:0.017691\n",
      "[102]\ttrain-mlogloss:0.00378\ttest-mlogloss:0.017698\n",
      "[103]\ttrain-mlogloss:0.003715\ttest-mlogloss:0.017687\n",
      "[104]\ttrain-mlogloss:0.003645\ttest-mlogloss:0.017686\n",
      "[105]\ttrain-mlogloss:0.003596\ttest-mlogloss:0.017687\n",
      "[106]\ttrain-mlogloss:0.003546\ttest-mlogloss:0.017721\n",
      "[107]\ttrain-mlogloss:0.003492\ttest-mlogloss:0.017719\n",
      "[108]\ttrain-mlogloss:0.003444\ttest-mlogloss:0.017703\n",
      "[109]\ttrain-mlogloss:0.003391\ttest-mlogloss:0.017715\n",
      "[110]\ttrain-mlogloss:0.003338\ttest-mlogloss:0.017704\n",
      "[111]\ttrain-mlogloss:0.003292\ttest-mlogloss:0.017697\n",
      "[112]\ttrain-mlogloss:0.003249\ttest-mlogloss:0.017702\n",
      "[113]\ttrain-mlogloss:0.003209\ttest-mlogloss:0.017696\n",
      "[114]\ttrain-mlogloss:0.003171\ttest-mlogloss:0.017697\n",
      "[115]\ttrain-mlogloss:0.003135\ttest-mlogloss:0.017687\n",
      "[116]\ttrain-mlogloss:0.003096\ttest-mlogloss:0.017687\n",
      "[117]\ttrain-mlogloss:0.003054\ttest-mlogloss:0.017686\n",
      "[118]\ttrain-mlogloss:0.003011\ttest-mlogloss:0.01768\n",
      "[119]\ttrain-mlogloss:0.002974\ttest-mlogloss:0.01768\n",
      "[120]\ttrain-mlogloss:0.002935\ttest-mlogloss:0.017675\n",
      "[121]\ttrain-mlogloss:0.002899\ttest-mlogloss:0.017706\n",
      "[122]\ttrain-mlogloss:0.002852\ttest-mlogloss:0.017705\n",
      "[123]\ttrain-mlogloss:0.00282\ttest-mlogloss:0.017733\n",
      "[124]\ttrain-mlogloss:0.002789\ttest-mlogloss:0.017736\n",
      "[125]\ttrain-mlogloss:0.00276\ttest-mlogloss:0.017756\n",
      "[126]\ttrain-mlogloss:0.00273\ttest-mlogloss:0.017744\n",
      "[127]\ttrain-mlogloss:0.002708\ttest-mlogloss:0.017752\n",
      "[128]\ttrain-mlogloss:0.002677\ttest-mlogloss:0.017781\n",
      "[129]\ttrain-mlogloss:0.002651\ttest-mlogloss:0.017774\n",
      "[130]\ttrain-mlogloss:0.002621\ttest-mlogloss:0.017782\n",
      "[131]\ttrain-mlogloss:0.002593\ttest-mlogloss:0.0178\n",
      "[132]\ttrain-mlogloss:0.002576\ttest-mlogloss:0.017816\n",
      "[133]\ttrain-mlogloss:0.002542\ttest-mlogloss:0.017856\n",
      "[134]\ttrain-mlogloss:0.002518\ttest-mlogloss:0.017881\n",
      "[135]\ttrain-mlogloss:0.002493\ttest-mlogloss:0.017881\n",
      "[136]\ttrain-mlogloss:0.002468\ttest-mlogloss:0.017893\n",
      "[137]\ttrain-mlogloss:0.002444\ttest-mlogloss:0.017883\n",
      "[138]\ttrain-mlogloss:0.002424\ttest-mlogloss:0.017901\n",
      "[139]\ttrain-mlogloss:0.002403\ttest-mlogloss:0.017924\n",
      "[140]\ttrain-mlogloss:0.002381\ttest-mlogloss:0.017953\n",
      "Stopping. Best iteration:\n",
      "[120]\ttrain-mlogloss:0.002935\ttest-mlogloss:0.017675\n",
      "\n"
     ]
    }
   ],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((116624, 97893), (116624,))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = sparse.hstack([train_data.iloc[:,:-1],tr_sparse1,tr_sparse2,tr_sparse3]).tocsr()\n",
    "train_X.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=0)\n",
    "pred_test_y,model=runLGB(x_train,y_train,x_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
